1 /*
2 * Copyright 2020 Broadcom
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 */
6
7 #define DT_DRV_COMPAT brcm_iproc_pax_dma_v1
8
9 #include <arch/cpu.h>
10 #include <cache.h>
11 #include <errno.h>
12 #include <init.h>
13 #include <kernel.h>
14 #include <linker/sections.h>
15 #include <soc.h>
16 #include <string.h>
17 #include <toolchain.h>
18 #include <zephyr/types.h>
19 #include <drivers/dma.h>
20 #include <drivers/pcie/endpoint/pcie_ep.h>
21 #include "dma_iproc_pax_v1.h"
22
23 #define LOG_LEVEL CONFIG_DMA_LOG_LEVEL
24 #include <logging/log.h>
25 LOG_MODULE_REGISTER(dma_iproc_pax);
26
27 #define PAX_DMA_DEV_NAME(dev) ((dev)->name)
28
29 #define PAX_DMA_DEV_CFG(dev) \
30 ((struct dma_iproc_pax_cfg *)(dev)->config)
31
32 #define PAX_DMA_DEV_DATA(dev) \
33 ((struct dma_iproc_pax_data *)(dev)->data)
34
35 /* Driver runtime data for PAX DMA and RM */
36 static struct dma_iproc_pax_data pax_dma_data;
37
reset_pkt_id(struct dma_iproc_pax_ring_data * ring)38 static inline uint32_t reset_pkt_id(struct dma_iproc_pax_ring_data *ring)
39 {
40 return ring->pkt_id = 0x0;
41 }
42
43 /**
44 * @brief Opaque/packet id allocator, range 0 to 31
45 */
alloc_pkt_id(struct dma_iproc_pax_ring_data * ring)46 static inline uint32_t alloc_pkt_id(struct dma_iproc_pax_ring_data *ring)
47 {
48 ring->pkt_id = (ring->pkt_id + 1) % 32;
49 return ring->pkt_id;
50 }
51
curr_pkt_id(struct dma_iproc_pax_ring_data * ring)52 static inline uint32_t curr_pkt_id(struct dma_iproc_pax_ring_data *ring)
53 {
54 return ring->pkt_id;
55 }
56
curr_toggle_val(struct dma_iproc_pax_ring_data * ring)57 static inline uint32_t curr_toggle_val(struct dma_iproc_pax_ring_data *ring)
58 {
59 return ring->curr.toggle;
60 }
61
62 /**
63 * @brief Populate header descriptor
64 */
rm_write_header_desc(void * desc,uint32_t toggle,uint32_t opq,uint32_t bdcount)65 static inline void rm_write_header_desc(void *desc, uint32_t toggle,
66 uint32_t opq, uint32_t bdcount)
67 {
68 struct rm_header *r = (struct rm_header *)desc;
69
70 r->opq = opq;
71 /* DMA descriptor count init vlaue */
72 r->bdcount = bdcount;
73 r->prot = 0x0;
74 /* No packet extension, start and end set to '1' */
75 r->start = 1;
76 r->end = 1;
77 r->toggle = toggle;
78 /* RM header type */
79 r->type = PAX_DMA_TYPE_RM_HEADER;
80 }
81
82 /**
83 * @brief Fill RM header descriptor for next transfer
84 * with invalid toggle
85 */
rm_write_header_next_desc(void * desc,struct dma_iproc_pax_ring_data * r,uint32_t opq,uint32_t bdcount)86 static inline void rm_write_header_next_desc(void *desc,
87 struct dma_iproc_pax_ring_data *r,
88 uint32_t opq, uint32_t bdcount)
89 {
90 /* Toggle bit is invalid until next payload configured */
91 rm_write_header_desc(desc, (r->curr.toggle == 0) ? 1 : 0, opq, bdcount);
92 }
93
rm_header_set_bd_count(void * desc,uint32_t bdcount)94 static inline void rm_header_set_bd_count(void *desc, uint32_t bdcount)
95 {
96 struct rm_header *r = (struct rm_header *)desc;
97
98 /* DMA descriptor count */
99 r->bdcount = bdcount;
100 }
101
rm_header_set_toggle(void * desc,uint32_t toggle)102 static inline void rm_header_set_toggle(void *desc, uint32_t toggle)
103 {
104 struct rm_header *r = (struct rm_header *)desc;
105
106 r->toggle = toggle;
107 }
108
109 /**
110 * @brief Populate dma header descriptor
111 */
rm_write_dma_header_desc(void * desc,struct dma_iproc_pax_payload * pl)112 static inline void rm_write_dma_header_desc(void *desc,
113 struct dma_iproc_pax_payload *pl)
114 {
115 struct dma_header_desc *hdr = (struct dma_header_desc *)desc;
116
117 hdr->length = pl->xfer_sz;
118 hdr->opcode = pl->direction;
119 /* DMA header type */
120 hdr->type = PAX_DMA_TYPE_DMA_DESC;
121 }
122
123 /**
124 * @brief Populate axi address descriptor
125 */
rm_write_axi_addr_desc(void * desc,struct dma_iproc_pax_payload * pl)126 static inline void rm_write_axi_addr_desc(void *desc,
127 struct dma_iproc_pax_payload *pl)
128 {
129 struct axi_addr_desc *axi = (struct axi_addr_desc *)desc;
130
131 axi->axi_addr = pl->axi_addr;
132 axi->type = PAX_DMA_TYPE_DMA_DESC;
133
134 }
135
136 /**
137 * @brief Populate pci address descriptor
138 */
rm_write_pci_addr_desc(void * desc,struct dma_iproc_pax_payload * pl)139 static inline void rm_write_pci_addr_desc(void *desc,
140 struct dma_iproc_pax_payload *pl)
141 {
142 struct pci_addr_desc *pci = (struct pci_addr_desc *)desc;
143
144 pci->pcie_addr = pl->pci_addr >> PAX_DMA_PCI_ADDR_ALIGNMT_SHIFT;
145 pci->type = PAX_DMA_TYPE_DMA_DESC;
146 }
147
148 /**
149 * @brief Return's pointer to the descriptor memory to be written next,
150 * skip next pointer descriptor address.
151 */
next_desc_addr(struct dma_iproc_pax_ring_data * ring)152 static void *next_desc_addr(struct dma_iproc_pax_ring_data *ring)
153 {
154 struct next_ptr_desc *nxt;
155 uintptr_t curr;
156
157 curr = (uintptr_t)ring->curr.write_ptr + PAX_DMA_RM_DESC_BDWIDTH;
158 /* if hit next table ptr, skip to next location, flip toggle */
159 nxt = (struct next_ptr_desc *)curr;
160 if (nxt->type == PAX_DMA_TYPE_NEXT_PTR) {
161 LOG_DBG("hit next_ptr@0x%lx:T%d, next_table@0x%lx\n",
162 curr, nxt->toggle, (uintptr_t)nxt->addr);
163 uintptr_t last = (uintptr_t)ring->bd +
164 PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
165 ring->curr.toggle = (ring->curr.toggle == 0) ? 1 : 0;
166 /* move to next addr, wrap around if hits end */
167 curr += PAX_DMA_RM_DESC_BDWIDTH;
168 if (curr == last) {
169 curr = (uintptr_t)ring->bd;
170 LOG_DBG("hit end of desc:0x%lx, wrap to 0x%lx\n",
171 last, curr);
172 }
173 }
174 ring->curr.write_ptr = (void *)curr;
175 return (void *)curr;
176 }
177
178 /**
179 * @brief Populate next ptr descriptor
180 */
rm_write_next_table_desc(void * desc,void * next_ptr,uint32_t toggle)181 static void rm_write_next_table_desc(void *desc, void *next_ptr,
182 uint32_t toggle)
183 {
184 struct next_ptr_desc *nxt = (struct next_ptr_desc *)desc;
185
186 nxt->addr = (uintptr_t)next_ptr;
187 nxt->type = PAX_DMA_TYPE_NEXT_PTR;
188 nxt->toggle = toggle;
189 }
190
prepare_ring(struct dma_iproc_pax_ring_data * ring)191 static void prepare_ring(struct dma_iproc_pax_ring_data *ring)
192 {
193 uintptr_t curr, next, last;
194 uint32_t toggle;
195 int buff_count = PAX_DMA_NUM_BD_BUFFS;
196
197 /* zero out descriptor area */
198 memset(ring->bd, 0x0, PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS);
199 memset(ring->cmpl, 0x0, PAX_DMA_RM_CMPL_RING_SIZE);
200
201 /* opaque/packet id value */
202 rm_write_header_desc(ring->bd, 0x0, reset_pkt_id(ring),
203 PAX_DMA_RM_DESC_BDCOUNT);
204 /* start with first buffer, valid toggle is 0x1 */
205 toggle = 0x1;
206 curr = (uintptr_t)ring->bd;
207 next = curr + PAX_DMA_RM_DESC_RING_SIZE;
208 last = curr + PAX_DMA_RM_DESC_RING_SIZE * PAX_DMA_NUM_BD_BUFFS;
209 do {
210 /* Place next_table desc as last BD entry on each buffer */
211 rm_write_next_table_desc(PAX_DMA_NEXT_TBL_ADDR((void *)curr),
212 (void *)next, toggle);
213
214 /* valid toggle flips for each buffer */
215 toggle = toggle ? 0x0 : 0x1;
216 curr += PAX_DMA_RM_DESC_RING_SIZE;
217 next += PAX_DMA_RM_DESC_RING_SIZE;
218 /* last entry, chain back to first buffer */
219 if (next == last) {
220 next = (uintptr_t)ring->bd;
221 }
222
223 } while (--buff_count);
224
225 dma_mb();
226
227 /* start programming from first RM header */
228 ring->curr.write_ptr = ring->bd;
229 /* valid toggle starts with 1 after reset */
230 ring->curr.toggle = 1;
231 /* completion read offset */
232 ring->curr.cmpl_rd_offs = 0;
233
234 /* init sync data for the ring */
235 ring->curr.sync_data.signature = PAX_DMA_WRITE_SYNC_SIGNATURE;
236 ring->curr.sync_data.ring = ring->idx;
237 /* pkt id for active dma xfer */
238 ring->curr.sync_data.opaque = 0x0;
239 /* pkt count for active dma xfer */
240 ring->curr.sync_data.total_pkts = 0x0;
241 }
242
init_rm(struct dma_iproc_pax_data * pd)243 static int init_rm(struct dma_iproc_pax_data *pd)
244 {
245 int ret = -ETIMEDOUT, timeout = 1000;
246
247 k_mutex_lock(&pd->dma_lock, K_FOREVER);
248 /* Wait for Ring Manager ready */
249 do {
250 LOG_DBG("Waiting for RM HW init\n");
251 if ((sys_read32(RM_COMM_REG(pd, RM_COMM_MAIN_HW_INIT_DONE)) &
252 RM_COMM_MAIN_HW_INIT_DONE_MASK)) {
253 ret = 0;
254 break;
255 }
256 k_sleep(K_MSEC(1));
257 } while (--timeout);
258 k_mutex_unlock(&pd->dma_lock);
259
260 if (!timeout) {
261 LOG_WRN("RM HW Init timedout!\n");
262 } else {
263 LOG_INF("PAX DMA RM HW Init Done\n");
264 }
265
266 return ret;
267 }
268
rm_cfg_start(struct dma_iproc_pax_data * pd)269 static void rm_cfg_start(struct dma_iproc_pax_data *pd)
270 {
271 uint32_t val;
272
273 k_mutex_lock(&pd->dma_lock, K_FOREVER);
274
275 /* set config done 0, enable toggle mode */
276 val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
277 val &= ~RM_COMM_CONTROL_CONFIG_DONE;
278 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
279
280 val &= ~(RM_COMM_CONTROL_MODE_MASK << RM_COMM_CONTROL_MODE_SHIFT);
281 val |= (RM_COMM_CONTROL_MODE_TOGGLE << RM_COMM_CONTROL_MODE_SHIFT);
282 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
283
284 /* Disable MSI */
285 sys_write32(RM_COMM_MSI_DISABLE_VAL,
286 RM_COMM_REG(pd, RM_COMM_MSI_DISABLE));
287 /* Enable Line interrupt */
288 val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
289 val |= RM_COMM_CONTROL_LINE_INTR_EN;
290 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
291
292 /* Enable AE_TIMEOUT */
293 sys_write32(RM_COMM_AE_TIMEOUT_VAL, RM_COMM_REG(pd,
294 RM_COMM_AE_TIMEOUT));
295 val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
296 val |= RM_COMM_CONTROL_AE_TIMEOUT_EN;
297 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
298
299 /* AE (Acceleration Engine) grouping to group '0' */
300 val = sys_read32(RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
301 val &= ~RM_AE_CTRL_AE_GROUP_MASK;
302 sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
303 val |= RM_AE_CONTROL_ACTIVE;
304 sys_write32(val, RM_COMM_REG(pd, RM_AE0_AE_CONTROL));
305
306 /* AXI read/write channel enable */
307 val = sys_read32(RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));
308 val |= (RM_COMM_AXI_CONTROL_RD_CH_EN | RM_COMM_AXI_CONTROL_WR_CH_EN);
309 sys_write32(val, RM_COMM_REG(pd, RM_COMM_AXI_CONTROL));
310
311 /* Tune RM control programming for 4 rings */
312 sys_write32(RM_COMM_TIMER_CONTROL0_VAL,
313 RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_0));
314 sys_write32(RM_COMM_TIMER_CONTROL1_VAL,
315 RM_COMM_REG(pd, RM_COMM_TIMER_CONTROL_1));
316 sys_write32(RM_COMM_RM_BURST_LENGTH,
317 RM_COMM_REG(pd, RM_COMM_RM_BURST_LENGTH));
318
319 /* Set Sequence max count to the max supported value */
320 val = sys_read32(RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));
321 val = (val | RING_MASK_SEQ_MAX_COUNT_MASK);
322 sys_write32(val, RM_COMM_REG(pd, RM_COMM_MASK_SEQUENCE_MAX_COUNT));
323
324 k_mutex_unlock(&pd->dma_lock);
325 }
326
rm_ring_clear_stats(struct dma_iproc_pax_data * pd,enum ring_idx idx)327 static void rm_ring_clear_stats(struct dma_iproc_pax_data *pd,
328 enum ring_idx idx)
329 {
330 /* Read ring Tx, Rx, and Outstanding counts to clear */
331 sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_LS));
332 sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_RECV_MS));
333 sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_LS));
334 sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_TRANS_MS));
335 sys_read32(RM_RING_REG(pd, idx, RING_NUM_REQ_OUTSTAND));
336 }
337
rm_cfg_finish(struct dma_iproc_pax_data * pd)338 static void rm_cfg_finish(struct dma_iproc_pax_data *pd)
339 {
340 uint32_t val;
341
342 k_mutex_lock(&pd->dma_lock, K_FOREVER);
343
344 /* set Ring config done */
345 val = sys_read32(RM_COMM_REG(pd, RM_COMM_CONTROL));
346 val |= RM_COMM_CONTROL_CONFIG_DONE;
347 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CONTROL));
348
349 k_mutex_unlock(&pd->dma_lock);
350 }
351
352 /* Activate/Deactivate rings */
set_ring_active(struct dma_iproc_pax_data * pd,enum ring_idx idx,bool active)353 static inline void set_ring_active(struct dma_iproc_pax_data *pd,
354 enum ring_idx idx,
355 bool active)
356 {
357 uint32_t val;
358
359 val = sys_read32(RM_RING_REG(pd, idx, RING_CONTROL));
360 if (active) {
361 val |= RING_CONTROL_ACTIVE;
362 } else {
363 val &= ~RING_CONTROL_ACTIVE;
364 }
365 sys_write32(val, RM_RING_REG(pd, idx, RING_CONTROL));
366 }
367
init_ring(struct dma_iproc_pax_data * pd,enum ring_idx idx)368 static int init_ring(struct dma_iproc_pax_data *pd, enum ring_idx idx)
369 {
370 uint32_t val;
371 uintptr_t desc = (uintptr_t)pd->ring[idx].bd;
372 uintptr_t cmpl = (uintptr_t)pd->ring[idx].cmpl;
373 int timeout = 5000, ret = 0;
374
375 k_mutex_lock(&pd->dma_lock, K_FOREVER);
376
377 /* Read cmpl write ptr incase previous dma stopped */
378 sys_read32(RM_RING_REG(pd, idx, RING_CMPL_WRITE_PTR));
379
380 /* Inactivate ring */
381 sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));
382
383 /* Flush ring before loading new descriptor */
384 sys_write32(RING_CONTROL_FLUSH, RM_RING_REG(pd, idx, RING_CONTROL));
385 do {
386 if (sys_read32(RM_RING_REG(pd, idx, RING_FLUSH_DONE)) &
387 RING_FLUSH_DONE_MASK) {
388 break;
389 }
390 k_busy_wait(1);
391 } while (--timeout);
392
393 if (!timeout) {
394 LOG_WRN("Ring %d flush timedout!\n", idx);
395 ret = -ETIMEDOUT;
396 goto err;
397 }
398
399 /* clear ring after flush */
400 sys_write32(0x0, RM_RING_REG(pd, idx, RING_CONTROL));
401
402 /* ring group id set to '0' */
403 val = sys_read32(RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));
404 val &= ~RING_COMM_CTRL_AE_GROUP_MASK;
405 sys_write32(val, RM_COMM_REG(pd, RM_COMM_CTRL_REG(idx)));
406
407 /* DDR update control, set timeout value */
408 val = RING_DDR_CONTROL_COUNT(RING_DDR_CONTROL_COUNT_VAL) |
409 RING_DDR_CONTROL_TIMER(RING_DDR_CONTROL_TIMER_VAL) |
410 RING_DDR_CONTROL_ENABLE;
411
412 sys_write32(val, RM_RING_REG(pd, idx, RING_CMPL_WR_PTR_DDR_CONTROL));
413
414 val = (uint32_t)((uintptr_t)desc >> PAX_DMA_RING_BD_ALIGN_ORDER);
415 sys_write32(val, RM_RING_REG(pd, idx, RING_BD_START_ADDR));
416 val = (uint32_t)((uintptr_t)cmpl >> PAX_DMA_RING_CMPL_ALIGN_ORDER);
417 sys_write32(val, RM_RING_REG(pd, idx, RING_CMPL_START_ADDR));
418 val = sys_read32(RM_RING_REG(pd, idx, RING_BD_READ_PTR));
419
420 /* keep ring inactive after init to avoid BD poll */
421 set_ring_active(pd, idx, false);
422 rm_ring_clear_stats(pd, idx);
423 err:
424 k_mutex_unlock(&pd->dma_lock);
425
426 return ret;
427 }
428
poll_on_write_sync(const struct device * dev,struct dma_iproc_pax_ring_data * ring)429 static int poll_on_write_sync(const struct device *dev,
430 struct dma_iproc_pax_ring_data *ring)
431 {
432 struct dma_iproc_pax_cfg *cfg = PAX_DMA_DEV_CFG(dev);
433 const struct device *pcidev;
434 struct dma_iproc_pax_write_sync_data sync_rd, *recv, *sent;
435 uint64_t pci_addr;
436 uint32_t *pci32, *axi32;
437 uint32_t zero_init = 0, timeout = PAX_DMA_MAX_SYNC_WAIT;
438 int ret;
439
440 pcidev = device_get_binding(cfg->pcie_dev_name);
441 if (!pcidev) {
442 LOG_ERR("Cannot get pcie device\n");
443 return -EINVAL;
444 }
445 recv = &sync_rd;
446 sent = &(ring->curr.sync_data);
447 /* form host pci sync address */
448 pci32 = (uint32_t *)&pci_addr;
449 pci32[0] = ring->sync_pci.addr_lo;
450 pci32[1] = ring->sync_pci.addr_hi;
451 axi32 = (uint32_t *)&sync_rd;
452
453 do {
454 ret = pcie_ep_xfer_data_memcpy(pcidev, pci_addr,
455 (uintptr_t *)axi32, 4,
456 PCIE_OB_LOWMEM, HOST_TO_DEVICE);
457
458 if (memcmp((void *)recv, (void *)sent, 4) == 0) {
459 /* clear the sync word */
460 ret = pcie_ep_xfer_data_memcpy(pcidev, pci_addr,
461 (uintptr_t *)&zero_init,
462 4, PCIE_OB_LOWMEM,
463 DEVICE_TO_HOST);
464 dma_mb();
465 ret = 0;
466 break;
467 }
468 k_busy_wait(1);
469 } while (--timeout);
470
471 if (!timeout) {
472 LOG_DBG("[ring %d]: not recvd write sync!\n", ring->idx);
473 ret = -ETIMEDOUT;
474 }
475
476 return ret;
477 }
478
process_cmpl_event(const struct device * dev,enum ring_idx idx,uint32_t pl_len)479 static int process_cmpl_event(const struct device *dev,
480 enum ring_idx idx, uint32_t pl_len)
481 {
482 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
483 uint32_t wr_offs, rd_offs, ret = 0;
484 struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);
485 struct cmpl_pkt *c;
486 uint32_t is_outstanding;
487
488 /* cmpl read offset, unprocessed cmpl location */
489 rd_offs = ring->curr.cmpl_rd_offs;
490
491 wr_offs = sys_read32(RM_RING_REG(pd, idx,
492 RING_CMPL_WRITE_PTR));
493
494 /* Update read ptr to "processed" */
495 ring->curr.cmpl_rd_offs = wr_offs;
496
497 /*
498 * Ensure consistency of completion descriptor
499 * The completion desc is updated by RM via AXI stream
500 * CPU need to ensure the memory operations are completed
501 * before reading cmpl area, by a "dsb"
502 * If Dcache enabled, need to invalidate the cachelines to
503 * read updated cmpl desc. The cache API also issues dsb.
504 */
505 dma_mb();
506
507 /* Decode cmpl pkt id to verify */
508 c = (struct cmpl_pkt *)((uintptr_t)ring->cmpl +
509 PAX_DMA_CMPL_DESC_SIZE * PAX_DMA_CURR_CMPL_IDX(wr_offs));
510
511 LOG_DBG("RING%d WR_PTR:%d opq:%d, rm_status:%x dma_status:%x\n",
512 idx, wr_offs, c->opq, c->rm_status, c->dma_status);
513
514 is_outstanding = sys_read32(RM_RING_REG(pd, idx,
515 RING_NUM_REQ_OUTSTAND));
516 if ((ring->curr.opq != c->opq) && (is_outstanding != 0)) {
517 LOG_ERR("RING%d: pkt id should be %d, rcvd %d outst=%d\n",
518 idx, ring->curr.opq, c->opq, is_outstanding);
519 ret = -EIO;
520 }
521 /* check for completion AE timeout */
522 if (c->rm_status == RM_COMPLETION_AE_TIMEOUT) {
523 LOG_ERR("RING%d WR_PTR:%d rm_status:%x AE Timeout!\n",
524 idx, wr_offs, c->rm_status);
525 /* TBD: Issue full card reset to restore operations */
526 LOG_ERR("Needs Card Reset to recover!\n");
527 ret = -ETIMEDOUT;
528 }
529
530 if (ring->dma_callback) {
531 ring->dma_callback(dev, ring->callback_arg, idx, ret);
532 }
533
534 return ret;
535 }
536
537 #ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
peek_ring_cmpl(const struct device * dev,enum ring_idx idx,uint32_t pl_len)538 static int peek_ring_cmpl(const struct device *dev,
539 enum ring_idx idx, uint32_t pl_len)
540 {
541 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
542 uint32_t wr_offs, rd_offs, timeout = PAX_DMA_MAX_POLL_WAIT;
543 struct dma_iproc_pax_ring_data *ring = &(pd->ring[idx]);
544
545 /* cmpl read offset, unprocessed cmpl location */
546 rd_offs = ring->curr.cmpl_rd_offs;
547
548 /* poll write_ptr until cmpl received for all buffers */
549 do {
550 wr_offs = sys_read32(RM_RING_REG(pd, idx,
551 RING_CMPL_WRITE_PTR));
552 if (PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs) >= pl_len)
553 break;
554 k_busy_wait(1);
555 } while (--timeout);
556
557 if (timeout == 0) {
558 LOG_ERR("RING%d timeout, rcvd %d, expected %d!\n",
559 idx, PAX_DMA_GET_CMPL_COUNT(wr_offs, rd_offs), pl_len);
560 /* More debug info on current dma instance */
561 LOG_ERR("WR_PTR:%x RD_PTR%x\n", wr_offs, rd_offs);
562 return -ETIMEDOUT;
563 }
564
565 return process_cmpl_event(dev, idx, pl_len);
566 }
567 #else
rm_isr(void * arg)568 static void rm_isr(void *arg)
569 {
570 uint32_t status, err_stat, idx;
571 const struct device *dev = arg;
572 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
573
574 /* read and clear interrupt status */
575 status = sys_read32(RM_COMM_REG(pd, RM_COMM_MSI_INTR_INTERRUPT_STATUS));
576 sys_write32(status, RM_COMM_REG(pd,
577 RM_COMM_MSI_INTERRUPT_STATUS_CLEAR));
578
579 /* read and clear DME/AE error interrupts */
580 err_stat = sys_read32(RM_COMM_REG(pd,
581 RM_COMM_DME_INTERRUPT_STATUS_MASK));
582 sys_write32(err_stat,
583 RM_COMM_REG(pd, RM_COMM_DME_INTERRUPT_STATUS_CLEAR));
584 err_stat =
585 sys_read32(RM_COMM_REG(pd,
586 RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_MASK));
587 sys_write32(err_stat,
588 RM_COMM_REG(pd,
589 RM_COMM_AE_INTERFACE_GROUP_0_INTERRUPT_CLEAR));
590
591 /* alert waiting thread to process, for each completed ring */
592 for (idx = PAX_DMA_RING0; idx < PAX_DMA_RINGS_MAX; idx++) {
593 if (status & (0x1 << idx)) {
594 k_sem_give(&pd->ring[idx].alert);
595 }
596 }
597 }
598 #endif
599
dma_iproc_pax_init(const struct device * dev)600 static int dma_iproc_pax_init(const struct device *dev)
601 {
602 struct dma_iproc_pax_cfg *cfg = PAX_DMA_DEV_CFG(dev);
603 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
604 int r;
605 uintptr_t mem_aligned;
606
607 pd->dma_base = cfg->dma_base;
608 pd->rm_comm_base = cfg->rm_comm_base;
609 pd->used_rings = (cfg->use_rings < PAX_DMA_RINGS_MAX) ?
610 cfg->use_rings : PAX_DMA_RINGS_MAX;
611
612 LOG_DBG("dma base:0x%x, rm comm base:0x%x, needed rings %d\n",
613 pd->dma_base, pd->rm_comm_base, pd->used_rings);
614
615 /* dma/rm access lock */
616 k_mutex_init(&pd->dma_lock);
617
618 /* Ring Manager H/W init */
619 if (init_rm(pd)) {
620 return -ETIMEDOUT;
621 }
622
623 /* common rm config */
624 rm_cfg_start(pd);
625
626 /* individual ring config */
627 for (r = 0; r < pd->used_rings; r++) {
628 /* per-ring mutex lock */
629 k_mutex_init(&pd->ring[r].lock);
630 /* Init alerts */
631 k_sem_init(&pd->ring[r].alert, 0, 1);
632
633 pd->ring[r].idx = r;
634 pd->ring[r].ring_base = cfg->rm_base +
635 PAX_DMA_RING_ADDR_OFFSET(r);
636 LOG_DBG("RING%d,VERSION:0x%x\n", pd->ring[r].idx,
637 sys_read32(RM_RING_REG(pd, r, RING_VER)));
638
639 /* Allocate for 2 BD buffers + cmpl buffer + payload struct */
640 pd->ring[r].ring_mem = (void *)((uintptr_t)cfg->bd_memory_base +
641 r *
642 PAX_DMA_PER_RING_ALLOC_SIZE);
643 if (!pd->ring[r].ring_mem) {
644 LOG_ERR("RING%d failed to alloc desc memory!\n", r);
645 return -ENOMEM;
646 }
647 /* Find 8K aligned address within allocated region */
648 mem_aligned = ((uintptr_t)pd->ring[r].ring_mem +
649 PAX_DMA_RING_ALIGN - 1) &
650 ~(PAX_DMA_RING_ALIGN - 1);
651
652 pd->ring[r].cmpl = (void *)mem_aligned;
653 pd->ring[r].bd = (void *)(mem_aligned +
654 PAX_DMA_RM_CMPL_RING_SIZE);
655 pd->ring[r].payload = (void *)((uintptr_t)pd->ring[r].bd +
656 PAX_DMA_RM_DESC_RING_SIZE *
657 PAX_DMA_NUM_BD_BUFFS);
658
659 LOG_DBG("Ring%d,allocated Mem:0x%p Size %d\n",
660 pd->ring[r].idx,
661 pd->ring[r].ring_mem,
662 PAX_DMA_PER_RING_ALLOC_SIZE);
663 LOG_DBG("Ring%d,BD:0x%p, CMPL:0x%p, PL:0x%p\n",
664 pd->ring[r].idx,
665 pd->ring[r].bd,
666 pd->ring[r].cmpl,
667 pd->ring[r].payload);
668
669 /* Prepare ring desc table */
670 prepare_ring(&(pd->ring[r]));
671
672 /* initialize ring */
673 init_ring(pd, r);
674 }
675
676 /* set ring config done */
677 rm_cfg_finish(pd);
678
679 #ifndef CONFIG_DMA_IPROC_PAX_POLL_MODE
680 /* Register and enable RM interrupt */
681 IRQ_CONNECT(DT_INST_IRQN(0),
682 DT_INST_IRQ(0, priority),
683 rm_isr,
684 DEVICE_DT_INST_GET(0),
685 0);
686 irq_enable(DT_INST_IRQN(0));
687 #else
688 LOG_INF("%s PAX DMA rings in poll mode!\n", PAX_DMA_DEV_NAME(dev));
689 #endif
690 LOG_INF("%s RM setup %d rings\n", PAX_DMA_DEV_NAME(dev),
691 pd->used_rings);
692
693 return 0;
694 }
695
696 #ifdef CONFIG_DMA_IPROC_PAX_POLL_MODE
set_pkt_count(const struct device * dev,enum ring_idx idx,uint32_t pl_len)697 static void set_pkt_count(const struct device *dev,
698 enum ring_idx idx,
699 uint32_t pl_len)
700 {
701 /* Nothing needs to be programmed here in poll mode */
702 }
703
wait_for_pkt_completion(const struct device * dev,enum ring_idx idx,uint32_t pl_len)704 static int wait_for_pkt_completion(const struct device *dev,
705 enum ring_idx idx,
706 uint32_t pl_len)
707 {
708 /* poll for completion */
709 return peek_ring_cmpl(dev, idx, pl_len + 1);
710 }
711 #else
set_pkt_count(const struct device * dev,enum ring_idx idx,uint32_t pl_len)712 static void set_pkt_count(const struct device *dev,
713 enum ring_idx idx,
714 uint32_t pl_len)
715 {
716 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
717 uint32_t val;
718
719 /* program packet count for interrupt assertion */
720 val = sys_read32(RM_RING_REG(pd, idx,
721 RING_CMPL_WR_PTR_DDR_CONTROL));
722 val &= ~RING_DDR_CONTROL_COUNT_MASK;
723 val |= RING_DDR_CONTROL_COUNT(pl_len);
724 sys_write32(val, RM_RING_REG(pd, idx,
725 RING_CMPL_WR_PTR_DDR_CONTROL));
726 }
727
wait_for_pkt_completion(const struct device * dev,enum ring_idx idx,uint32_t pl_len)728 static int wait_for_pkt_completion(const struct device *dev,
729 enum ring_idx idx,
730 uint32_t pl_len)
731 {
732 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
733 struct dma_iproc_pax_ring_data *ring;
734
735 ring = &(pd->ring[idx]);
736
737 /* wait for sg dma completion alert */
738 if (k_sem_take(&ring->alert, K_MSEC(PAX_DMA_TIMEOUT)) != 0) {
739 LOG_ERR("PAX DMA [ring %d] Timeout!\n", idx);
740 return -ETIMEDOUT;
741 }
742
743 return process_cmpl_event(dev, idx, pl_len);
744 }
745 #endif
746
dma_iproc_pax_do_xfer(const struct device * dev,enum ring_idx idx,struct dma_iproc_pax_payload * pl,uint32_t pl_len)747 static int dma_iproc_pax_do_xfer(const struct device *dev,
748 enum ring_idx idx,
749 struct dma_iproc_pax_payload *pl,
750 uint32_t pl_len)
751 {
752 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
753 struct dma_iproc_pax_cfg *cfg = PAX_DMA_DEV_CFG(dev);
754 int ret = 0, cnt;
755 struct dma_iproc_pax_ring_data *ring;
756 void *hdr;
757 uint32_t toggle_bit;
758 struct dma_iproc_pax_payload sync_pl;
759 struct dma_iproc_pax_addr64 sync;
760
761 ring = &(pd->ring[idx]);
762 pl = ring->payload;
763
764 /*
765 * Host sync buffer isn't ready at zephyr/driver init-time
766 * Read the host address location once at first DMA write
767 * on that ring.
768 */
769 if ((ring->sync_pci.addr_lo == 0x0) &&
770 (ring->sync_pci.addr_hi == 0x0)) {
771 /* populate sync data location */
772 LOG_DBG("sync addr loc 0x%x\n", cfg->scr_addr_loc);
773
774 sync.addr_lo = sys_read32(cfg->scr_addr_loc
775 + 4);
776 sync.addr_hi = sys_read32(cfg->scr_addr_loc);
777 ring->sync_pci.addr_lo = sync.addr_lo + idx * 4;
778 ring->sync_pci.addr_hi = sync.addr_hi;
779 LOG_DBG("ring:%d,sync addr:0x%x.0x%x\n", idx,
780 ring->sync_pci.addr_hi,
781 ring->sync_pci.addr_lo);
782 }
783
784 /* account extra sync packet */
785 ring->curr.sync_data.opaque = ring->curr.opq;
786 ring->curr.sync_data.total_pkts = pl_len;
787 memcpy((void *)&ring->sync_loc,
788 (void *)&(ring->curr.sync_data), 4);
789 sync_pl.pci_addr = ring->sync_pci.addr_lo |
790 (uint64_t)ring->sync_pci.addr_hi << 32;
791 sync_pl.axi_addr = (uintptr_t)&ring->sync_loc;
792
793 sync_pl.xfer_sz = 4; /* 4-bytes */
794 sync_pl.direction = CARD_TO_HOST;
795
796 /* Get descriptor write pointer for first header */
797 hdr = (void *)ring->curr.write_ptr;
798 /* current toggle bit */
799 toggle_bit = ring->curr.toggle;
800 /* current opq value for cmpl check */
801 ring->curr.opq = curr_pkt_id(ring);
802
803 /* DMA desc count for first payload */
804 rm_header_set_bd_count(hdr, PAX_DMA_RM_DESC_BDCOUNT);
805
806 /* Form dma descriptors for total sg payload */
807 for (cnt = 0; cnt < pl_len; cnt++) {
808 rm_write_dma_header_desc(next_desc_addr(ring), pl + cnt);
809 rm_write_axi_addr_desc(next_desc_addr(ring), pl + cnt);
810 rm_write_pci_addr_desc(next_desc_addr(ring), pl + cnt);
811 /* Toggle may flip, program updated toggle value */
812 rm_write_header_desc(next_desc_addr(ring),
813 curr_toggle_val(ring),
814 curr_pkt_id(ring),
815 PAX_DMA_RM_DESC_BDCOUNT);
816 }
817
818 /* Append write sync payload descriptors */
819 rm_write_dma_header_desc(next_desc_addr(ring), &sync_pl);
820 rm_write_axi_addr_desc(next_desc_addr(ring), &sync_pl);
821 rm_write_pci_addr_desc(next_desc_addr(ring), &sync_pl);
822
823 /* RM header for next transfer, RM wait on (invalid) toggle bit */
824 rm_write_header_next_desc(next_desc_addr(ring), ring, alloc_pkt_id(ring),
825 PAX_DMA_RM_DESC_BDCOUNT);
826
827 set_pkt_count(dev, idx, pl_len + 1);
828
829 /* Ensure memory write before toggle flip */
830 dma_mb();
831
832 /* set toggle to valid in first header */
833 rm_header_set_toggle(hdr, toggle_bit);
834 /* activate the ring */
835 set_ring_active(pd, idx, true);
836
837 ret = wait_for_pkt_completion(dev, idx, pl_len + 1);
838 if (ret)
839 goto err_ret;
840
841 ret = poll_on_write_sync(dev, ring);
842 k_mutex_lock(&ring->lock, K_FOREVER);
843 ring->ring_active = 0;
844 k_mutex_unlock(&ring->lock);
845
846 err_ret:
847 ring->ring_active = 0;
848 /* deactivate the ring until next active transfer */
849 set_ring_active(pd, idx, false);
850
851 return ret;
852 }
853
dma_iproc_pax_configure(const struct device * dev,uint32_t channel,struct dma_config * cfg)854 static int dma_iproc_pax_configure(const struct device *dev, uint32_t channel,
855 struct dma_config *cfg)
856 {
857 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
858 struct dma_iproc_pax_ring_data *ring;
859 uint32_t xfer_sz;
860 int ret = 0;
861 #ifdef CONFIG_DMA_IPROC_PAX_DEBUG
862 uint32_t *pci_addr32;
863 uint32_t *axi_addr32;
864 #endif
865
866 if (channel >= PAX_DMA_RINGS_MAX) {
867 LOG_ERR("Invalid ring/channel %d\n", channel);
868 return -EINVAL;
869 }
870
871 ring = &(pd->ring[channel]);
872 k_mutex_lock(&ring->lock, K_FOREVER);
873
874 if (cfg->block_count > 1) {
875 /* Scatter/gather list handling is not supported */
876 ret = -ENOTSUP;
877 goto err;
878 }
879
880 if (ring->ring_active) {
881 ret = -EBUSY;
882 goto err;
883 }
884
885 ring->ring_active = 1;
886
887 if (cfg->channel_direction == MEMORY_TO_PERIPHERAL) {
888 #ifdef CONFIG_DMA_IPROC_PAX_DEBUG
889 axi_addr32 = (uint32_t *)&cfg->head_block->source_address;
890 pci_addr32 = (uint32_t *)&cfg->head_block->dest_address;
891 #endif
892 ring->payload->direction = CARD_TO_HOST;
893 ring->payload->pci_addr = cfg->head_block->dest_address;
894 ring->payload->axi_addr = cfg->head_block->source_address;
895 } else if (cfg->channel_direction == PERIPHERAL_TO_MEMORY) {
896 #ifdef CONFIG_DMA_IPROC_PAX_DEBUG
897 axi_addr32 = (uint32_t *)&cfg->head_block->dest_address;
898 pci_addr32 = (uint32_t *)&cfg->head_block->source_address;
899 #endif
900 ring->payload->direction = HOST_TO_CARD;
901 ring->payload->pci_addr = cfg->head_block->source_address;
902 ring->payload->axi_addr = cfg->head_block->dest_address;
903 } else {
904 ring->ring_active = 0;
905 ret = -ENOTSUP;
906 goto err;
907 }
908
909 xfer_sz = cfg->head_block->block_size;
910
911 #ifdef CONFIG_DMA_IPROC_PAX_DEBUG
912 if (xfer_sz > PAX_DMA_MAX_SIZE) {
913 LOG_ERR("Unsupported size: %d\n", xfer_size);
914 ring->ring_active = 0;
915 ret = -EINVAL;
916 goto err;
917 }
918
919 if (xfer_sz % PAX_DMA_MIN_SIZE) {
920 LOG_ERR("Unaligned size 0x%x\n", xfer_size);
921 ring->ring_active = 0;
922 ret = -EINVAL;
923 goto err;
924 }
925
926 if (pci_addr32[0] % PAX_DMA_ADDR_ALIGN) {
927 LOG_ERR("Unaligned Host addr: 0x%x.0x%x\n",
928 pci_addr32[1], pci_addr32[0]);
929 ring->ring_active = 0;
930 ret = -EINVAL;
931 goto err;
932 }
933
934 if (axi_addr32[0] % PAX_DMA_ADDR_ALIGN) {
935 LOG_ERR("Unaligned Card addr: 0x%x.0x%x\n",
936 axi_addr32[1], axi_addr32[0]);
937 ring->ring_active = 0;
938 ret = -EINVAL;
939 goto err;
940 }
941 #endif
942 ring->payload->xfer_sz = xfer_sz;
943
944 ring->dma_callback = cfg->dma_callback;
945 ring->callback_arg = cfg->user_data;
946 err:
947 k_mutex_unlock(&ring->lock);
948
949 return ret;
950 }
951
dma_iproc_pax_transfer_start(const struct device * dev,uint32_t channel)952 static int dma_iproc_pax_transfer_start(const struct device *dev,
953 uint32_t channel)
954 {
955 int ret = 0;
956 struct dma_iproc_pax_data *pd = PAX_DMA_DEV_DATA(dev);
957 struct dma_iproc_pax_ring_data *ring;
958
959 if (channel >= PAX_DMA_RINGS_MAX) {
960 LOG_ERR("Invalid ring %d\n", channel);
961 return -EINVAL;
962 }
963 ring = &(pd->ring[channel]);
964 /* do dma transfer of single buffer */
965 ret = dma_iproc_pax_do_xfer(dev, channel, ring->payload, 1);
966
967 return ret;
968 }
969
dma_iproc_pax_transfer_stop(const struct device * dev,uint32_t channel)970 static int dma_iproc_pax_transfer_stop(const struct device *dev,
971 uint32_t channel)
972 {
973 return 0;
974 }
975
976 static const struct dma_driver_api pax_dma_driver_api = {
977 .config = dma_iproc_pax_configure,
978 .start = dma_iproc_pax_transfer_start,
979 .stop = dma_iproc_pax_transfer_stop,
980 };
981
982 static const struct dma_iproc_pax_cfg pax_dma_cfg = {
983 .dma_base = DT_INST_REG_ADDR_BY_NAME(0, dme_regs),
984 .rm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_ring_regs),
985 .rm_comm_base = DT_INST_REG_ADDR_BY_NAME(0, rm_comm_regs),
986 .use_rings = DT_INST_PROP(0, dma_channels),
987 .bd_memory_base = (void *)DT_INST_PROP_BY_IDX(0, bd_memory, 0),
988 .scr_addr_loc = DT_INST_PROP(0, scr_addr_loc),
989 .pcie_dev_name = DT_INST_PROP_BY_PHANDLE(0, pcie_ep, label),
990 };
991
992 DEVICE_DT_INST_DEFINE(0,
993 &dma_iproc_pax_init,
994 NULL,
995 &pax_dma_data,
996 &pax_dma_cfg,
997 POST_KERNEL,
998 CONFIG_KERNEL_INIT_PRIORITY_DEVICE,
999 &pax_dma_driver_api);
1000
