1 /*
2 * Qualcomm Technologies HIDMA DMA engine low level code
3 *
4 * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 and
8 * only version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16 #include <linux/dmaengine.h>
17 #include <linux/slab.h>
18 #include <linux/interrupt.h>
19 #include <linux/mm.h>
20 #include <linux/highmem.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/delay.h>
23 #include <linux/atomic.h>
24 #include <linux/iopoll.h>
25 #include <linux/kfifo.h>
26 #include <linux/bitops.h>
27
28 #include "hidma.h"
29
30 #define HIDMA_EVRE_SIZE 16 /* each EVRE is 16 bytes */
31
32 #define HIDMA_TRCA_CTRLSTS_REG 0x000
33 #define HIDMA_TRCA_RING_LOW_REG 0x008
34 #define HIDMA_TRCA_RING_HIGH_REG 0x00C
35 #define HIDMA_TRCA_RING_LEN_REG 0x010
36 #define HIDMA_TRCA_DOORBELL_REG 0x400
37
38 #define HIDMA_EVCA_CTRLSTS_REG 0x000
39 #define HIDMA_EVCA_INTCTRL_REG 0x004
40 #define HIDMA_EVCA_RING_LOW_REG 0x008
41 #define HIDMA_EVCA_RING_HIGH_REG 0x00C
42 #define HIDMA_EVCA_RING_LEN_REG 0x010
43 #define HIDMA_EVCA_WRITE_PTR_REG 0x020
44 #define HIDMA_EVCA_DOORBELL_REG 0x400
45
46 #define HIDMA_EVCA_IRQ_STAT_REG 0x100
47 #define HIDMA_EVCA_IRQ_CLR_REG 0x108
48 #define HIDMA_EVCA_IRQ_EN_REG 0x110
49
50 #define HIDMA_EVRE_CFG_IDX 0
51
52 #define HIDMA_EVRE_ERRINFO_BIT_POS 24
53 #define HIDMA_EVRE_CODE_BIT_POS 28
54
55 #define HIDMA_EVRE_ERRINFO_MASK GENMASK(3, 0)
56 #define HIDMA_EVRE_CODE_MASK GENMASK(3, 0)
57
58 #define HIDMA_CH_CONTROL_MASK GENMASK(7, 0)
59 #define HIDMA_CH_STATE_MASK GENMASK(7, 0)
60 #define HIDMA_CH_STATE_BIT_POS 0x8
61
62 #define HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS 0
63 #define HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS 1
64 #define HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS 9
65 #define HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS 10
66 #define HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS 11
67 #define HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS 14
68
69 #define ENABLE_IRQS (BIT(HIDMA_IRQ_EV_CH_EOB_IRQ_BIT_POS) | \
70 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
71 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
72 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
73 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS) | \
74 BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS))
75
76 #define HIDMA_INCREMENT_ITERATOR(iter, size, ring_size) \
77 do { \
78 iter += size; \
79 if (iter >= ring_size) \
80 iter -= ring_size; \
81 } while (0)
82
83 #define HIDMA_CH_STATE(val) \
84 ((val >> HIDMA_CH_STATE_BIT_POS) & HIDMA_CH_STATE_MASK)
85
86 #define HIDMA_ERR_INT_MASK \
87 (BIT(HIDMA_IRQ_TR_CH_INVALID_TRE_BIT_POS) | \
88 BIT(HIDMA_IRQ_TR_CH_TRE_RD_RSP_ER_BIT_POS) | \
89 BIT(HIDMA_IRQ_EV_CH_WR_RESP_BIT_POS) | \
90 BIT(HIDMA_IRQ_TR_CH_DATA_RD_ER_BIT_POS) | \
91 BIT(HIDMA_IRQ_TR_CH_DATA_WR_ER_BIT_POS))
92
93 enum ch_command {
94 HIDMA_CH_DISABLE = 0,
95 HIDMA_CH_ENABLE = 1,
96 HIDMA_CH_SUSPEND = 2,
97 HIDMA_CH_RESET = 9,
98 };
99
100 enum ch_state {
101 HIDMA_CH_DISABLED = 0,
102 HIDMA_CH_ENABLED = 1,
103 HIDMA_CH_RUNNING = 2,
104 HIDMA_CH_SUSPENDED = 3,
105 HIDMA_CH_STOPPED = 4,
106 };
107
108 enum err_code {
109 HIDMA_EVRE_STATUS_COMPLETE = 1,
110 HIDMA_EVRE_STATUS_ERROR = 4,
111 };
112
hidma_is_chan_enabled(int state)113 static int hidma_is_chan_enabled(int state)
114 {
115 switch (state) {
116 case HIDMA_CH_ENABLED:
117 case HIDMA_CH_RUNNING:
118 return true;
119 default:
120 return false;
121 }
122 }
123
hidma_ll_free(struct hidma_lldev * lldev,u32 tre_ch)124 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)
125 {
126 struct hidma_tre *tre;
127
128 if (tre_ch >= lldev->nr_tres) {
129 dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);
130 return;
131 }
132
133 tre = &lldev->trepool[tre_ch];
134 if (atomic_read(&tre->allocated) != true) {
135 dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);
136 return;
137 }
138
139 atomic_set(&tre->allocated, 0);
140 }
141
hidma_ll_request(struct hidma_lldev * lldev,u32 sig,const char * dev_name,void (* callback)(void * data),void * data,u32 * tre_ch)142 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,
143 void (*callback)(void *data), void *data, u32 *tre_ch)
144 {
145 unsigned int i;
146 struct hidma_tre *tre;
147 u32 *tre_local;
148
149 if (!tre_ch || !lldev)
150 return -EINVAL;
151
152 /* need to have at least one empty spot in the queue */
153 for (i = 0; i < lldev->nr_tres - 1; i++) {
154 if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))
155 break;
156 }
157
158 if (i == (lldev->nr_tres - 1))
159 return -ENOMEM;
160
161 tre = &lldev->trepool[i];
162 tre->dma_sig = sig;
163 tre->dev_name = dev_name;
164 tre->callback = callback;
165 tre->data = data;
166 tre->idx = i;
167 tre->status = 0;
168 tre->queued = 0;
169 tre->err_code = 0;
170 tre->err_info = 0;
171 tre->lldev = lldev;
172 tre_local = &tre->tre_local[0];
173 tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;
174 tre_local[HIDMA_TRE_CFG_IDX] |= BIT(16); /* set IEOB */
175 *tre_ch = i;
176 if (callback)
177 callback(data);
178 return 0;
179 }
180
181 /*
182 * Multiple TREs may be queued and waiting in the pending queue.
183 */
hidma_ll_tre_complete(unsigned long arg)184 static void hidma_ll_tre_complete(unsigned long arg)
185 {
186 struct hidma_lldev *lldev = (struct hidma_lldev *)arg;
187 struct hidma_tre *tre;
188
189 while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {
190 /* call the user if it has been read by the hardware */
191 if (tre->callback)
192 tre->callback(tre->data);
193 }
194 }
195
hidma_post_completed(struct hidma_lldev * lldev,u8 err_info,u8 err_code)196 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,
197 u8 err_code)
198 {
199 struct hidma_tre *tre;
200 unsigned long flags;
201 u32 tre_iterator;
202
203 spin_lock_irqsave(&lldev->lock, flags);
204
205 tre_iterator = lldev->tre_processed_off;
206 tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];
207 if (!tre) {
208 spin_unlock_irqrestore(&lldev->lock, flags);
209 dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",
210 tre_iterator / HIDMA_TRE_SIZE);
211 return -EINVAL;
212 }
213 lldev->pending_tre_list[tre->tre_index] = NULL;
214
215 /*
216 * Keep track of pending TREs that SW is expecting to receive
217 * from HW. We got one now. Decrement our counter.
218 */
219 if (atomic_dec_return(&lldev->pending_tre_count) < 0) {
220 dev_warn(lldev->dev, "tre count mismatch on completion");
221 atomic_set(&lldev->pending_tre_count, 0);
222 }
223
224 HIDMA_INCREMENT_ITERATOR(tre_iterator, HIDMA_TRE_SIZE,
225 lldev->tre_ring_size);
226 lldev->tre_processed_off = tre_iterator;
227 spin_unlock_irqrestore(&lldev->lock, flags);
228
229 tre->err_info = err_info;
230 tre->err_code = err_code;
231 tre->queued = 0;
232
233 kfifo_put(&lldev->handoff_fifo, tre);
234 tasklet_schedule(&lldev->task);
235
236 return 0;
237 }
238
239 /*
240 * Called to handle the interrupt for the channel.
241 * Return a positive number if TRE or EVRE were consumed on this run.
242 * Return a positive number if there are pending TREs or EVREs.
243 * Return 0 if there is nothing to consume or no pending TREs/EVREs found.
244 */
hidma_handle_tre_completion(struct hidma_lldev * lldev)245 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)
246 {
247 u32 evre_ring_size = lldev->evre_ring_size;
248 u32 err_info, err_code, evre_write_off;
249 u32 evre_iterator;
250 u32 num_completed = 0;
251
252 evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
253 evre_iterator = lldev->evre_processed_off;
254
255 if ((evre_write_off > evre_ring_size) ||
256 (evre_write_off % HIDMA_EVRE_SIZE)) {
257 dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");
258 return 0;
259 }
260
261 /*
262 * By the time control reaches here the number of EVREs and TREs
263 * may not match. Only consume the ones that hardware told us.
264 */
265 while ((evre_iterator != evre_write_off)) {
266 u32 *current_evre = lldev->evre_ring + evre_iterator;
267 u32 cfg;
268
269 cfg = current_evre[HIDMA_EVRE_CFG_IDX];
270 err_info = cfg >> HIDMA_EVRE_ERRINFO_BIT_POS;
271 err_info &= HIDMA_EVRE_ERRINFO_MASK;
272 err_code =
273 (cfg >> HIDMA_EVRE_CODE_BIT_POS) & HIDMA_EVRE_CODE_MASK;
274
275 if (hidma_post_completed(lldev, err_info, err_code))
276 break;
277
278 HIDMA_INCREMENT_ITERATOR(evre_iterator, HIDMA_EVRE_SIZE,
279 evre_ring_size);
280
281 /*
282 * Read the new event descriptor written by the HW.
283 * As we are processing the delivered events, other events
284 * get queued to the SW for processing.
285 */
286 evre_write_off =
287 readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);
288 num_completed++;
289
290 /*
291 * An error interrupt might have arrived while we are processing
292 * the completed interrupt.
293 */
294 if (!hidma_ll_isenabled(lldev))
295 break;
296 }
297
298 if (num_completed) {
299 u32 evre_read_off = (lldev->evre_processed_off +
300 HIDMA_EVRE_SIZE * num_completed);
301 evre_read_off = evre_read_off % evre_ring_size;
302 writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);
303
304 /* record the last processed tre offset */
305 lldev->evre_processed_off = evre_read_off;
306 }
307
308 return num_completed;
309 }
310
hidma_cleanup_pending_tre(struct hidma_lldev * lldev,u8 err_info,u8 err_code)311 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,
312 u8 err_code)
313 {
314 while (atomic_read(&lldev->pending_tre_count)) {
315 if (hidma_post_completed(lldev, err_info, err_code))
316 break;
317 }
318 }
319
hidma_ll_reset(struct hidma_lldev * lldev)320 static int hidma_ll_reset(struct hidma_lldev *lldev)
321 {
322 u32 val;
323 int ret;
324
325 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
326 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
327 val |= HIDMA_CH_RESET << 16;
328 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
329
330 /*
331 * Delay 10ms after reset to allow DMA logic to quiesce.
332 * Do a polled read up to 1ms and 10ms maximum.
333 */
334 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
335 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
336 1000, 10000);
337 if (ret) {
338 dev_err(lldev->dev, "transfer channel did not reset\n");
339 return ret;
340 }
341
342 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
343 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
344 val |= HIDMA_CH_RESET << 16;
345 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
346
347 /*
348 * Delay 10ms after reset to allow DMA logic to quiesce.
349 * Do a polled read up to 1ms and 10ms maximum.
350 */
351 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
352 HIDMA_CH_STATE(val) == HIDMA_CH_DISABLED,
353 1000, 10000);
354 if (ret)
355 return ret;
356
357 lldev->trch_state = HIDMA_CH_DISABLED;
358 lldev->evch_state = HIDMA_CH_DISABLED;
359 return 0;
360 }
361
362 /*
363 * The interrupt handler for HIDMA will try to consume as many pending
364 * EVRE from the event queue as possible. Each EVRE has an associated
365 * TRE that holds the user interface parameters. EVRE reports the
366 * result of the transaction. Hardware guarantees ordering between EVREs
367 * and TREs. We use last processed offset to figure out which TRE is
368 * associated with which EVRE. If two TREs are consumed by HW, the EVREs
369 * are in order in the event ring.
370 *
371 * This handler will do a one pass for consuming EVREs. Other EVREs may
372 * be delivered while we are working. It will try to consume incoming
373 * EVREs one more time and return.
374 *
375 * For unprocessed EVREs, hardware will trigger another interrupt until
376 * all the interrupt bits are cleared.
377 *
378 * Hardware guarantees that by the time interrupt is observed, all data
379 * transactions in flight are delivered to their respective places and
380 * are visible to the CPU.
381 *
382 * On demand paging for IOMMU is only supported for PCIe via PRI
383 * (Page Request Interface) not for HIDMA. All other hardware instances
384 * including HIDMA work on pinned DMA addresses.
385 *
386 * HIDMA is not aware of IOMMU presence since it follows the DMA API. All
387 * IOMMU latency will be built into the data movement time. By the time
388 * interrupt happens, IOMMU lookups + data movement has already taken place.
389 *
390 * While the first read in a typical PCI endpoint ISR flushes all outstanding
391 * requests traditionally to the destination, this concept does not apply
392 * here for this HW.
393 */
hidma_ll_int_handler_internal(struct hidma_lldev * lldev,int cause)394 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)
395 {
396 unsigned long irqflags;
397
398 if (cause & HIDMA_ERR_INT_MASK) {
399 dev_err(lldev->dev, "error 0x%x, disabling...\n",
400 cause);
401
402 /* Clear out pending interrupts */
403 writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
404
405 /* No further submissions. */
406 hidma_ll_disable(lldev);
407
408 /* Driver completes the txn and intimates the client.*/
409 hidma_cleanup_pending_tre(lldev, 0xFF,
410 HIDMA_EVRE_STATUS_ERROR);
411
412 return;
413 }
414
415 spin_lock_irqsave(&lldev->lock, irqflags);
416 writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
417 spin_unlock_irqrestore(&lldev->lock, irqflags);
418
419 /*
420 * Fine tuned for this HW...
421 *
422 * This ISR has been designed for this particular hardware. Relaxed
423 * read and write accessors are used for performance reasons due to
424 * interrupt delivery guarantees. Do not copy this code blindly and
425 * expect that to work.
426 *
427 * Try to consume as many EVREs as possible.
428 */
429 hidma_handle_tre_completion(lldev);
430 }
431
hidma_ll_inthandler(int chirq,void * arg)432 irqreturn_t hidma_ll_inthandler(int chirq, void *arg)
433 {
434 struct hidma_lldev *lldev = arg;
435 u32 status;
436 u32 enable;
437 u32 cause;
438
439 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
440 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
441 cause = status & enable;
442
443 while (cause) {
444 hidma_ll_int_handler_internal(lldev, cause);
445
446 /*
447 * Another interrupt might have arrived while we are
448 * processing this one. Read the new cause.
449 */
450 status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
451 enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
452 cause = status & enable;
453 }
454
455 return IRQ_HANDLED;
456 }
457
hidma_ll_inthandler_msi(int chirq,void * arg,int cause)458 irqreturn_t hidma_ll_inthandler_msi(int chirq, void *arg, int cause)
459 {
460 struct hidma_lldev *lldev = arg;
461
462 hidma_ll_int_handler_internal(lldev, cause);
463 return IRQ_HANDLED;
464 }
465
hidma_ll_enable(struct hidma_lldev * lldev)466 int hidma_ll_enable(struct hidma_lldev *lldev)
467 {
468 u32 val;
469 int ret;
470
471 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
472 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
473 val |= HIDMA_CH_ENABLE << 16;
474 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
475
476 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
477 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
478 1000, 10000);
479 if (ret) {
480 dev_err(lldev->dev, "event channel did not get enabled\n");
481 return ret;
482 }
483
484 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
485 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
486 val |= HIDMA_CH_ENABLE << 16;
487 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
488
489 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
490 hidma_is_chan_enabled(HIDMA_CH_STATE(val)),
491 1000, 10000);
492 if (ret) {
493 dev_err(lldev->dev, "transfer channel did not get enabled\n");
494 return ret;
495 }
496
497 lldev->trch_state = HIDMA_CH_ENABLED;
498 lldev->evch_state = HIDMA_CH_ENABLED;
499
500 /* enable irqs */
501 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
502
503 return 0;
504 }
505
hidma_ll_start(struct hidma_lldev * lldev)506 void hidma_ll_start(struct hidma_lldev *lldev)
507 {
508 unsigned long irqflags;
509
510 spin_lock_irqsave(&lldev->lock, irqflags);
511 writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);
512 spin_unlock_irqrestore(&lldev->lock, irqflags);
513 }
514
hidma_ll_isenabled(struct hidma_lldev * lldev)515 bool hidma_ll_isenabled(struct hidma_lldev *lldev)
516 {
517 u32 val;
518
519 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
520 lldev->trch_state = HIDMA_CH_STATE(val);
521 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
522 lldev->evch_state = HIDMA_CH_STATE(val);
523
524 /* both channels have to be enabled before calling this function */
525 if (hidma_is_chan_enabled(lldev->trch_state) &&
526 hidma_is_chan_enabled(lldev->evch_state))
527 return true;
528
529 return false;
530 }
531
hidma_ll_queue_request(struct hidma_lldev * lldev,u32 tre_ch)532 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)
533 {
534 struct hidma_tre *tre;
535 unsigned long flags;
536
537 tre = &lldev->trepool[tre_ch];
538
539 /* copy the TRE into its location in the TRE ring */
540 spin_lock_irqsave(&lldev->lock, flags);
541 tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;
542 lldev->pending_tre_list[tre->tre_index] = tre;
543 memcpy(lldev->tre_ring + lldev->tre_write_offset,
544 &tre->tre_local[0], HIDMA_TRE_SIZE);
545 tre->err_code = 0;
546 tre->err_info = 0;
547 tre->queued = 1;
548 atomic_inc(&lldev->pending_tre_count);
549 lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)
550 % lldev->tre_ring_size;
551 spin_unlock_irqrestore(&lldev->lock, flags);
552 }
553
554 /*
555 * Note that even though we stop this channel if there is a pending transaction
556 * in flight it will complete and follow the callback. This request will
557 * prevent further requests to be made.
558 */
hidma_ll_disable(struct hidma_lldev * lldev)559 int hidma_ll_disable(struct hidma_lldev *lldev)
560 {
561 u32 val;
562 int ret;
563
564 /* The channel needs to be in working state */
565 if (!hidma_ll_isenabled(lldev))
566 return 0;
567
568 val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
569 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
570 val |= HIDMA_CH_SUSPEND << 16;
571 writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);
572
573 /*
574 * Start the wait right after the suspend is confirmed.
575 * Do a polled read up to 1ms and 10ms maximum.
576 */
577 ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,
578 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
579 1000, 10000);
580 if (ret)
581 return ret;
582
583 val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
584 val &= ~(HIDMA_CH_CONTROL_MASK << 16);
585 val |= HIDMA_CH_SUSPEND << 16;
586 writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);
587
588 /*
589 * Start the wait right after the suspend is confirmed
590 * Delay up to 10ms after reset to allow DMA logic to quiesce.
591 */
592 ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,
593 HIDMA_CH_STATE(val) == HIDMA_CH_SUSPENDED,
594 1000, 10000);
595 if (ret)
596 return ret;
597
598 lldev->trch_state = HIDMA_CH_SUSPENDED;
599 lldev->evch_state = HIDMA_CH_SUSPENDED;
600
601 /* disable interrupts */
602 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
603 return 0;
604 }
605
hidma_ll_set_transfer_params(struct hidma_lldev * lldev,u32 tre_ch,dma_addr_t src,dma_addr_t dest,u32 len,u32 flags,u32 txntype)606 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,
607 dma_addr_t src, dma_addr_t dest, u32 len,
608 u32 flags, u32 txntype)
609 {
610 struct hidma_tre *tre;
611 u32 *tre_local;
612
613 if (tre_ch >= lldev->nr_tres) {
614 dev_err(lldev->dev, "invalid TRE number in transfer params:%d",
615 tre_ch);
616 return;
617 }
618
619 tre = &lldev->trepool[tre_ch];
620 if (atomic_read(&tre->allocated) != true) {
621 dev_err(lldev->dev, "trying to set params on an unused TRE:%d",
622 tre_ch);
623 return;
624 }
625
626 tre_local = &tre->tre_local[0];
627 tre_local[HIDMA_TRE_CFG_IDX] &= ~GENMASK(7, 0);
628 tre_local[HIDMA_TRE_CFG_IDX] |= txntype;
629 tre_local[HIDMA_TRE_LEN_IDX] = len;
630 tre_local[HIDMA_TRE_SRC_LOW_IDX] = lower_32_bits(src);
631 tre_local[HIDMA_TRE_SRC_HI_IDX] = upper_32_bits(src);
632 tre_local[HIDMA_TRE_DEST_LOW_IDX] = lower_32_bits(dest);
633 tre_local[HIDMA_TRE_DEST_HI_IDX] = upper_32_bits(dest);
634 tre->int_flags = flags;
635 }
636
637 /*
638 * Called during initialization and after an error condition
639 * to restore hardware state.
640 */
hidma_ll_setup(struct hidma_lldev * lldev)641 int hidma_ll_setup(struct hidma_lldev *lldev)
642 {
643 int rc;
644 u64 addr;
645 u32 val;
646 u32 nr_tres = lldev->nr_tres;
647
648 atomic_set(&lldev->pending_tre_count, 0);
649 lldev->tre_processed_off = 0;
650 lldev->evre_processed_off = 0;
651 lldev->tre_write_offset = 0;
652
653 /* disable interrupts */
654 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
655
656 /* clear all pending interrupts */
657 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
658 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
659
660 rc = hidma_ll_reset(lldev);
661 if (rc)
662 return rc;
663
664 /*
665 * Clear all pending interrupts again.
666 * Otherwise, we observe reset complete interrupts.
667 */
668 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
669 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
670
671 /* disable interrupts again after reset */
672 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
673
674 addr = lldev->tre_dma;
675 writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);
676 writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);
677 writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);
678
679 addr = lldev->evre_dma;
680 writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);
681 writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);
682 writel(HIDMA_EVRE_SIZE * nr_tres,
683 lldev->evca + HIDMA_EVCA_RING_LEN_REG);
684
685 /* configure interrupts */
686 hidma_ll_setup_irq(lldev, lldev->msi_support);
687
688 rc = hidma_ll_enable(lldev);
689 if (rc)
690 return rc;
691
692 return rc;
693 }
694
hidma_ll_setup_irq(struct hidma_lldev * lldev,bool msi)695 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)
696 {
697 u32 val;
698
699 lldev->msi_support = msi;
700
701 /* disable interrupts again after reset */
702 writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
703 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
704
705 /* support IRQ by default */
706 val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);
707 val &= ~0xF;
708 if (!lldev->msi_support)
709 val = val | 0x1;
710 writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);
711
712 /* clear all pending interrupts and enable them */
713 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
714 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
715 }
716
hidma_ll_init(struct device * dev,u32 nr_tres,void __iomem * trca,void __iomem * evca,u8 chidx)717 struct hidma_lldev *hidma_ll_init(struct device *dev, u32 nr_tres,
718 void __iomem *trca, void __iomem *evca,
719 u8 chidx)
720 {
721 u32 required_bytes;
722 struct hidma_lldev *lldev;
723 int rc;
724 size_t sz;
725
726 if (!trca || !evca || !dev || !nr_tres)
727 return NULL;
728
729 /* need at least four TREs */
730 if (nr_tres < 4)
731 return NULL;
732
733 /* need an extra space */
734 nr_tres += 1;
735
736 lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);
737 if (!lldev)
738 return NULL;
739
740 lldev->evca = evca;
741 lldev->trca = trca;
742 lldev->dev = dev;
743 sz = sizeof(struct hidma_tre);
744 lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);
745 if (!lldev->trepool)
746 return NULL;
747
748 required_bytes = sizeof(lldev->pending_tre_list[0]);
749 lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,
750 GFP_KERNEL);
751 if (!lldev->pending_tre_list)
752 return NULL;
753
754 sz = (HIDMA_TRE_SIZE + 1) * nr_tres;
755 lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,
756 GFP_KERNEL);
757 if (!lldev->tre_ring)
758 return NULL;
759
760 memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres);
761 lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;
762 lldev->nr_tres = nr_tres;
763
764 /* the TRE ring has to be TRE_SIZE aligned */
765 if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {
766 u8 tre_ring_shift;
767
768 tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;
769 tre_ring_shift = HIDMA_TRE_SIZE - tre_ring_shift;
770 lldev->tre_dma += tre_ring_shift;
771 lldev->tre_ring += tre_ring_shift;
772 }
773
774 sz = (HIDMA_EVRE_SIZE + 1) * nr_tres;
775 lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,
776 GFP_KERNEL);
777 if (!lldev->evre_ring)
778 return NULL;
779
780 memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres);
781 lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;
782
783 /* the EVRE ring has to be EVRE_SIZE aligned */
784 if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {
785 u8 evre_ring_shift;
786
787 evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;
788 evre_ring_shift = HIDMA_EVRE_SIZE - evre_ring_shift;
789 lldev->evre_dma += evre_ring_shift;
790 lldev->evre_ring += evre_ring_shift;
791 }
792 lldev->nr_tres = nr_tres;
793 lldev->chidx = chidx;
794
795 sz = nr_tres * sizeof(struct hidma_tre *);
796 rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);
797 if (rc)
798 return NULL;
799
800 rc = hidma_ll_setup(lldev);
801 if (rc)
802 return NULL;
803
804 spin_lock_init(&lldev->lock);
805 tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev);
806 lldev->initialized = 1;
807 writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
808 return lldev;
809 }
810
hidma_ll_uninit(struct hidma_lldev * lldev)811 int hidma_ll_uninit(struct hidma_lldev *lldev)
812 {
813 u32 required_bytes;
814 int rc = 0;
815 u32 val;
816
817 if (!lldev)
818 return -ENODEV;
819
820 if (!lldev->initialized)
821 return 0;
822
823 lldev->initialized = 0;
824
825 required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;
826 tasklet_kill(&lldev->task);
827 memset(lldev->trepool, 0, required_bytes);
828 lldev->trepool = NULL;
829 atomic_set(&lldev->pending_tre_count, 0);
830 lldev->tre_write_offset = 0;
831
832 rc = hidma_ll_reset(lldev);
833
834 /*
835 * Clear all pending interrupts again.
836 * Otherwise, we observe reset complete interrupts.
837 */
838 val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);
839 writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);
840 writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);
841 return rc;
842 }
843
hidma_ll_status(struct hidma_lldev * lldev,u32 tre_ch)844 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)
845 {
846 enum dma_status ret = DMA_ERROR;
847 struct hidma_tre *tre;
848 unsigned long flags;
849 u8 err_code;
850
851 spin_lock_irqsave(&lldev->lock, flags);
852
853 tre = &lldev->trepool[tre_ch];
854 err_code = tre->err_code;
855
856 if (err_code & HIDMA_EVRE_STATUS_COMPLETE)
857 ret = DMA_COMPLETE;
858 else if (err_code & HIDMA_EVRE_STATUS_ERROR)
859 ret = DMA_ERROR;
860 else
861 ret = DMA_IN_PROGRESS;
862 spin_unlock_irqrestore(&lldev->lock, flags);
863
864 return ret;
865 }
866
