1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Filename: dma.c
4 *
5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6 * Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7 *
8 * (C) Copyright 2013 IBM Corporation
9 */
10
11 #include <linux/slab.h>
12 #include "rsxx_priv.h"
13
14 struct rsxx_dma {
15 struct list_head list;
16 u8 cmd;
17 unsigned int laddr; /* Logical address */
18 struct {
19 u32 off;
20 u32 cnt;
21 } sub_page;
22 dma_addr_t dma_addr;
23 struct page *page;
24 unsigned int pg_off; /* Page Offset */
25 rsxx_dma_cb cb;
26 void *cb_data;
27 };
28
29 /* This timeout is used to detect a stalled DMA channel */
30 #define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
31
32 struct hw_status {
33 u8 status;
34 u8 tag;
35 __le16 count;
36 __le32 _rsvd2;
37 __le64 _rsvd3;
38 } __packed;
39
40 enum rsxx_dma_status {
41 DMA_SW_ERR = 0x1,
42 DMA_HW_FAULT = 0x2,
43 DMA_CANCELLED = 0x4,
44 };
45
46 struct hw_cmd {
47 u8 command;
48 u8 tag;
49 u8 _rsvd;
50 u8 sub_page; /* Bit[0:2]: 512byte offset */
51 /* Bit[4:6]: 512byte count */
52 __le32 device_addr;
53 __le64 host_addr;
54 } __packed;
55
56 enum rsxx_hw_cmd {
57 HW_CMD_BLK_DISCARD = 0x70,
58 HW_CMD_BLK_WRITE = 0x80,
59 HW_CMD_BLK_READ = 0xC0,
60 HW_CMD_BLK_RECON_READ = 0xE0,
61 };
62
63 enum rsxx_hw_status {
64 HW_STATUS_CRC = 0x01,
65 HW_STATUS_HARD_ERR = 0x02,
66 HW_STATUS_SOFT_ERR = 0x04,
67 HW_STATUS_FAULT = 0x08,
68 };
69
70 static struct kmem_cache *rsxx_dma_pool;
71
72 struct dma_tracker {
73 int next_tag;
74 struct rsxx_dma *dma;
75 };
76
77 #define DMA_TRACKER_LIST_SIZE8 (sizeof(struct dma_tracker_list) + \
78 (sizeof(struct dma_tracker) * RSXX_MAX_OUTSTANDING_CMDS))
79
80 struct dma_tracker_list {
81 spinlock_t lock;
82 int head;
83 struct dma_tracker list[];
84 };
85
86
87 /*----------------- Misc Utility Functions -------------------*/
rsxx_addr8_to_laddr(u64 addr8,struct rsxx_cardinfo * card)88 static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
89 {
90 unsigned long long tgt_addr8;
91
92 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
93 card->_stripe.upper_mask) |
94 ((addr8) & card->_stripe.lower_mask);
95 do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
96 return tgt_addr8;
97 }
98
rsxx_get_dma_tgt(struct rsxx_cardinfo * card,u64 addr8)99 static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
100 {
101 unsigned int tgt;
102
103 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
104
105 return tgt;
106 }
107
rsxx_dma_queue_reset(struct rsxx_cardinfo * card)108 void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
109 {
110 /* Reset all DMA Command/Status Queues */
111 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
112 }
113
get_dma_size(struct rsxx_dma * dma)114 static unsigned int get_dma_size(struct rsxx_dma *dma)
115 {
116 if (dma->sub_page.cnt)
117 return dma->sub_page.cnt << 9;
118 else
119 return RSXX_HW_BLK_SIZE;
120 }
121
122
123 /*----------------- DMA Tracker -------------------*/
set_tracker_dma(struct dma_tracker_list * trackers,int tag,struct rsxx_dma * dma)124 static void set_tracker_dma(struct dma_tracker_list *trackers,
125 int tag,
126 struct rsxx_dma *dma)
127 {
128 trackers->list[tag].dma = dma;
129 }
130
get_tracker_dma(struct dma_tracker_list * trackers,int tag)131 static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
132 int tag)
133 {
134 return trackers->list[tag].dma;
135 }
136
pop_tracker(struct dma_tracker_list * trackers)137 static int pop_tracker(struct dma_tracker_list *trackers)
138 {
139 int tag;
140
141 spin_lock(&trackers->lock);
142 tag = trackers->head;
143 if (tag != -1) {
144 trackers->head = trackers->list[tag].next_tag;
145 trackers->list[tag].next_tag = -1;
146 }
147 spin_unlock(&trackers->lock);
148
149 return tag;
150 }
151
push_tracker(struct dma_tracker_list * trackers,int tag)152 static void push_tracker(struct dma_tracker_list *trackers, int tag)
153 {
154 spin_lock(&trackers->lock);
155 trackers->list[tag].next_tag = trackers->head;
156 trackers->head = tag;
157 trackers->list[tag].dma = NULL;
158 spin_unlock(&trackers->lock);
159 }
160
161
162 /*----------------- Interrupt Coalescing -------------*/
163 /*
164 * Interrupt Coalescing Register Format:
165 * Interrupt Timer (64ns units) [15:0]
166 * Interrupt Count [24:16]
167 * Reserved [31:25]
168 */
169 #define INTR_COAL_LATENCY_MASK (0x0000ffff)
170
171 #define INTR_COAL_COUNT_SHIFT 16
172 #define INTR_COAL_COUNT_BITS 9
173 #define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
174 INTR_COAL_COUNT_SHIFT)
175 #define INTR_COAL_LATENCY_UNITS_NS 64
176
177
dma_intr_coal_val(u32 mode,u32 count,u32 latency)178 static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
179 {
180 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
181
182 if (mode == RSXX_INTR_COAL_DISABLED)
183 return 0;
184
185 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
186 (latency_units & INTR_COAL_LATENCY_MASK);
187
188 }
189
dma_intr_coal_auto_tune(struct rsxx_cardinfo * card)190 static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
191 {
192 int i;
193 u32 q_depth = 0;
194 u32 intr_coal;
195
196 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
197 unlikely(card->eeh_state))
198 return;
199
200 for (i = 0; i < card->n_targets; i++)
201 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
202
203 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
204 q_depth / 2,
205 card->config.data.intr_coal.latency);
206 iowrite32(intr_coal, card->regmap + INTR_COAL);
207 }
208
209 /*----------------- RSXX DMA Handling -------------------*/
rsxx_free_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma)210 static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
211 {
212 if (dma->cmd != HW_CMD_BLK_DISCARD) {
213 if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
214 dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr,
215 get_dma_size(dma),
216 dma->cmd == HW_CMD_BLK_WRITE ?
217 DMA_TO_DEVICE :
218 DMA_FROM_DEVICE);
219 }
220 }
221
222 kmem_cache_free(rsxx_dma_pool, dma);
223 }
224
rsxx_complete_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma,unsigned int status)225 static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
226 struct rsxx_dma *dma,
227 unsigned int status)
228 {
229 if (status & DMA_SW_ERR)
230 ctrl->stats.dma_sw_err++;
231 if (status & DMA_HW_FAULT)
232 ctrl->stats.dma_hw_fault++;
233 if (status & DMA_CANCELLED)
234 ctrl->stats.dma_cancelled++;
235
236 if (dma->cb)
237 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
238
239 rsxx_free_dma(ctrl, dma);
240 }
241
rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl * ctrl,struct list_head * q,unsigned int done)242 int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
243 struct list_head *q, unsigned int done)
244 {
245 struct rsxx_dma *dma;
246 struct rsxx_dma *tmp;
247 int cnt = 0;
248
249 list_for_each_entry_safe(dma, tmp, q, list) {
250 list_del(&dma->list);
251 if (done & COMPLETE_DMA)
252 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
253 else
254 rsxx_free_dma(ctrl, dma);
255 cnt++;
256 }
257
258 return cnt;
259 }
260
rsxx_requeue_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma)261 static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
262 struct rsxx_dma *dma)
263 {
264 /*
265 * Requeued DMAs go to the front of the queue so they are issued
266 * first.
267 */
268 spin_lock_bh(&ctrl->queue_lock);
269 ctrl->stats.sw_q_depth++;
270 list_add(&dma->list, &ctrl->queue);
271 spin_unlock_bh(&ctrl->queue_lock);
272 }
273
rsxx_handle_dma_error(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma,u8 hw_st)274 static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
275 struct rsxx_dma *dma,
276 u8 hw_st)
277 {
278 unsigned int status = 0;
279 int requeue_cmd = 0;
280
281 dev_dbg(CARD_TO_DEV(ctrl->card),
282 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
283 dma->cmd, dma->laddr, hw_st);
284
285 if (hw_st & HW_STATUS_CRC)
286 ctrl->stats.crc_errors++;
287 if (hw_st & HW_STATUS_HARD_ERR)
288 ctrl->stats.hard_errors++;
289 if (hw_st & HW_STATUS_SOFT_ERR)
290 ctrl->stats.soft_errors++;
291
292 switch (dma->cmd) {
293 case HW_CMD_BLK_READ:
294 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
295 if (ctrl->card->scrub_hard) {
296 dma->cmd = HW_CMD_BLK_RECON_READ;
297 requeue_cmd = 1;
298 ctrl->stats.reads_retried++;
299 } else {
300 status |= DMA_HW_FAULT;
301 ctrl->stats.reads_failed++;
302 }
303 } else if (hw_st & HW_STATUS_FAULT) {
304 status |= DMA_HW_FAULT;
305 ctrl->stats.reads_failed++;
306 }
307
308 break;
309 case HW_CMD_BLK_RECON_READ:
310 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
311 /* Data could not be reconstructed. */
312 status |= DMA_HW_FAULT;
313 ctrl->stats.reads_failed++;
314 }
315
316 break;
317 case HW_CMD_BLK_WRITE:
318 status |= DMA_HW_FAULT;
319 ctrl->stats.writes_failed++;
320
321 break;
322 case HW_CMD_BLK_DISCARD:
323 status |= DMA_HW_FAULT;
324 ctrl->stats.discards_failed++;
325
326 break;
327 default:
328 dev_err(CARD_TO_DEV(ctrl->card),
329 "Unknown command in DMA!(cmd: x%02x "
330 "laddr x%08x st: x%02x\n",
331 dma->cmd, dma->laddr, hw_st);
332 status |= DMA_SW_ERR;
333
334 break;
335 }
336
337 if (requeue_cmd)
338 rsxx_requeue_dma(ctrl, dma);
339 else
340 rsxx_complete_dma(ctrl, dma, status);
341 }
342
dma_engine_stalled(struct timer_list * t)343 static void dma_engine_stalled(struct timer_list *t)
344 {
345 struct rsxx_dma_ctrl *ctrl = from_timer(ctrl, t, activity_timer);
346 int cnt;
347
348 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
349 unlikely(ctrl->card->eeh_state))
350 return;
351
352 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
353 /*
354 * The dma engine was stalled because the SW_CMD_IDX write
355 * was lost. Issue it again to recover.
356 */
357 dev_warn(CARD_TO_DEV(ctrl->card),
358 "SW_CMD_IDX write was lost, re-writing...\n");
359 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
360 mod_timer(&ctrl->activity_timer,
361 jiffies + DMA_ACTIVITY_TIMEOUT);
362 } else {
363 dev_warn(CARD_TO_DEV(ctrl->card),
364 "DMA channel %d has stalled, faulting interface.\n",
365 ctrl->id);
366 ctrl->card->dma_fault = 1;
367
368 /* Clean up the DMA queue */
369 spin_lock(&ctrl->queue_lock);
370 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
371 spin_unlock(&ctrl->queue_lock);
372
373 cnt += rsxx_dma_cancel(ctrl);
374
375 if (cnt)
376 dev_info(CARD_TO_DEV(ctrl->card),
377 "Freed %d queued DMAs on channel %d\n",
378 cnt, ctrl->id);
379 }
380 }
381
rsxx_issue_dmas(struct rsxx_dma_ctrl * ctrl)382 static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
383 {
384 struct rsxx_dma *dma;
385 int tag;
386 int cmds_pending = 0;
387 struct hw_cmd *hw_cmd_buf;
388 int dir;
389
390 hw_cmd_buf = ctrl->cmd.buf;
391
392 if (unlikely(ctrl->card->halt) ||
393 unlikely(ctrl->card->eeh_state))
394 return;
395
396 while (1) {
397 spin_lock_bh(&ctrl->queue_lock);
398 if (list_empty(&ctrl->queue)) {
399 spin_unlock_bh(&ctrl->queue_lock);
400 break;
401 }
402 spin_unlock_bh(&ctrl->queue_lock);
403
404 tag = pop_tracker(ctrl->trackers);
405 if (tag == -1)
406 break;
407
408 spin_lock_bh(&ctrl->queue_lock);
409 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
410 list_del(&dma->list);
411 ctrl->stats.sw_q_depth--;
412 spin_unlock_bh(&ctrl->queue_lock);
413
414 /*
415 * This will catch any DMAs that slipped in right before the
416 * fault, but was queued after all the other DMAs were
417 * cancelled.
418 */
419 if (unlikely(ctrl->card->dma_fault)) {
420 push_tracker(ctrl->trackers, tag);
421 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
422 continue;
423 }
424
425 if (dma->cmd != HW_CMD_BLK_DISCARD) {
426 if (dma->cmd == HW_CMD_BLK_WRITE)
427 dir = DMA_TO_DEVICE;
428 else
429 dir = DMA_FROM_DEVICE;
430
431 /*
432 * The function dma_map_page is placed here because we
433 * can only, by design, issue up to 255 commands to the
434 * hardware at one time per DMA channel. So the maximum
435 * amount of mapped memory would be 255 * 4 channels *
436 * 4096 Bytes which is less than 2GB, the limit of a x8
437 * Non-HWWD PCIe slot. This way the dma_map_page
438 * function should never fail because of a lack of
439 * mappable memory.
440 */
441 dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page,
442 dma->pg_off, dma->sub_page.cnt << 9, dir);
443 if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
444 push_tracker(ctrl->trackers, tag);
445 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
446 continue;
447 }
448 }
449
450 set_tracker_dma(ctrl->trackers, tag, dma);
451 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
452 hw_cmd_buf[ctrl->cmd.idx].tag = tag;
453 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
454 hw_cmd_buf[ctrl->cmd.idx].sub_page =
455 ((dma->sub_page.cnt & 0x7) << 4) |
456 (dma->sub_page.off & 0x7);
457
458 hw_cmd_buf[ctrl->cmd.idx].device_addr =
459 cpu_to_le32(dma->laddr);
460
461 hw_cmd_buf[ctrl->cmd.idx].host_addr =
462 cpu_to_le64(dma->dma_addr);
463
464 dev_dbg(CARD_TO_DEV(ctrl->card),
465 "Issue DMA%d(laddr %d tag %d) to idx %d\n",
466 ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
467
468 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
469 cmds_pending++;
470
471 if (dma->cmd == HW_CMD_BLK_WRITE)
472 ctrl->stats.writes_issued++;
473 else if (dma->cmd == HW_CMD_BLK_DISCARD)
474 ctrl->stats.discards_issued++;
475 else
476 ctrl->stats.reads_issued++;
477 }
478
479 /* Let HW know we've queued commands. */
480 if (cmds_pending) {
481 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
482 mod_timer(&ctrl->activity_timer,
483 jiffies + DMA_ACTIVITY_TIMEOUT);
484
485 if (unlikely(ctrl->card->eeh_state)) {
486 del_timer_sync(&ctrl->activity_timer);
487 return;
488 }
489
490 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
491 }
492 }
493
rsxx_dma_done(struct rsxx_dma_ctrl * ctrl)494 static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
495 {
496 struct rsxx_dma *dma;
497 unsigned long flags;
498 u16 count;
499 u8 status;
500 u8 tag;
501 struct hw_status *hw_st_buf;
502
503 hw_st_buf = ctrl->status.buf;
504
505 if (unlikely(ctrl->card->halt) ||
506 unlikely(ctrl->card->dma_fault) ||
507 unlikely(ctrl->card->eeh_state))
508 return;
509
510 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
511
512 while (count == ctrl->e_cnt) {
513 /*
514 * The read memory-barrier is necessary to keep aggressive
515 * processors/optimizers (such as the PPC Apple G5) from
516 * reordering the following status-buffer tag & status read
517 * *before* the count read on subsequent iterations of the
518 * loop!
519 */
520 rmb();
521
522 status = hw_st_buf[ctrl->status.idx].status;
523 tag = hw_st_buf[ctrl->status.idx].tag;
524
525 dma = get_tracker_dma(ctrl->trackers, tag);
526 if (dma == NULL) {
527 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
528 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
529 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
530
531 dev_err(CARD_TO_DEV(ctrl->card),
532 "No tracker for tag %d "
533 "(idx %d id %d)\n",
534 tag, ctrl->status.idx, ctrl->id);
535 return;
536 }
537
538 dev_dbg(CARD_TO_DEV(ctrl->card),
539 "Completing DMA%d"
540 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
541 ctrl->id, dma->laddr, tag, status, count,
542 ctrl->status.idx);
543
544 atomic_dec(&ctrl->stats.hw_q_depth);
545
546 mod_timer(&ctrl->activity_timer,
547 jiffies + DMA_ACTIVITY_TIMEOUT);
548
549 if (status)
550 rsxx_handle_dma_error(ctrl, dma, status);
551 else
552 rsxx_complete_dma(ctrl, dma, 0);
553
554 push_tracker(ctrl->trackers, tag);
555
556 ctrl->status.idx = (ctrl->status.idx + 1) &
557 RSXX_CS_IDX_MASK;
558 ctrl->e_cnt++;
559
560 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
561 }
562
563 dma_intr_coal_auto_tune(ctrl->card);
564
565 if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
566 del_timer_sync(&ctrl->activity_timer);
567
568 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
569 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
570 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
571
572 spin_lock_bh(&ctrl->queue_lock);
573 if (ctrl->stats.sw_q_depth)
574 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
575 spin_unlock_bh(&ctrl->queue_lock);
576 }
577
rsxx_schedule_issue(struct work_struct * work)578 static void rsxx_schedule_issue(struct work_struct *work)
579 {
580 struct rsxx_dma_ctrl *ctrl;
581
582 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
583
584 mutex_lock(&ctrl->work_lock);
585 rsxx_issue_dmas(ctrl);
586 mutex_unlock(&ctrl->work_lock);
587 }
588
rsxx_schedule_done(struct work_struct * work)589 static void rsxx_schedule_done(struct work_struct *work)
590 {
591 struct rsxx_dma_ctrl *ctrl;
592
593 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
594
595 mutex_lock(&ctrl->work_lock);
596 rsxx_dma_done(ctrl);
597 mutex_unlock(&ctrl->work_lock);
598 }
599
rsxx_queue_discard(struct rsxx_cardinfo * card,struct list_head * q,unsigned int laddr,rsxx_dma_cb cb,void * cb_data)600 static blk_status_t rsxx_queue_discard(struct rsxx_cardinfo *card,
601 struct list_head *q,
602 unsigned int laddr,
603 rsxx_dma_cb cb,
604 void *cb_data)
605 {
606 struct rsxx_dma *dma;
607
608 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
609 if (!dma)
610 return BLK_STS_RESOURCE;
611
612 dma->cmd = HW_CMD_BLK_DISCARD;
613 dma->laddr = laddr;
614 dma->dma_addr = 0;
615 dma->sub_page.off = 0;
616 dma->sub_page.cnt = 0;
617 dma->page = NULL;
618 dma->pg_off = 0;
619 dma->cb = cb;
620 dma->cb_data = cb_data;
621
622 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
623
624 list_add_tail(&dma->list, q);
625
626 return 0;
627 }
628
rsxx_queue_dma(struct rsxx_cardinfo * card,struct list_head * q,int dir,unsigned int dma_off,unsigned int dma_len,unsigned int laddr,struct page * page,unsigned int pg_off,rsxx_dma_cb cb,void * cb_data)629 static blk_status_t rsxx_queue_dma(struct rsxx_cardinfo *card,
630 struct list_head *q,
631 int dir,
632 unsigned int dma_off,
633 unsigned int dma_len,
634 unsigned int laddr,
635 struct page *page,
636 unsigned int pg_off,
637 rsxx_dma_cb cb,
638 void *cb_data)
639 {
640 struct rsxx_dma *dma;
641
642 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
643 if (!dma)
644 return BLK_STS_RESOURCE;
645
646 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
647 dma->laddr = laddr;
648 dma->sub_page.off = (dma_off >> 9);
649 dma->sub_page.cnt = (dma_len >> 9);
650 dma->page = page;
651 dma->pg_off = pg_off;
652 dma->cb = cb;
653 dma->cb_data = cb_data;
654
655 dev_dbg(CARD_TO_DEV(card),
656 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
657 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
658 dma->sub_page.cnt, dma->page, dma->pg_off);
659
660 /* Queue the DMA */
661 list_add_tail(&dma->list, q);
662
663 return 0;
664 }
665
rsxx_dma_queue_bio(struct rsxx_cardinfo * card,struct bio * bio,atomic_t * n_dmas,rsxx_dma_cb cb,void * cb_data)666 blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
667 struct bio *bio,
668 atomic_t *n_dmas,
669 rsxx_dma_cb cb,
670 void *cb_data)
671 {
672 struct list_head dma_list[RSXX_MAX_TARGETS];
673 struct bio_vec bvec;
674 struct bvec_iter iter;
675 unsigned long long addr8;
676 unsigned int laddr;
677 unsigned int bv_len;
678 unsigned int bv_off;
679 unsigned int dma_off;
680 unsigned int dma_len;
681 int dma_cnt[RSXX_MAX_TARGETS];
682 int tgt;
683 blk_status_t st;
684 int i;
685
686 addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
687 atomic_set(n_dmas, 0);
688
689 for (i = 0; i < card->n_targets; i++) {
690 INIT_LIST_HEAD(&dma_list[i]);
691 dma_cnt[i] = 0;
692 }
693
694 if (bio_op(bio) == REQ_OP_DISCARD) {
695 bv_len = bio->bi_iter.bi_size;
696
697 while (bv_len > 0) {
698 tgt = rsxx_get_dma_tgt(card, addr8);
699 laddr = rsxx_addr8_to_laddr(addr8, card);
700
701 st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
702 cb, cb_data);
703 if (st)
704 goto bvec_err;
705
706 dma_cnt[tgt]++;
707 atomic_inc(n_dmas);
708 addr8 += RSXX_HW_BLK_SIZE;
709 bv_len -= RSXX_HW_BLK_SIZE;
710 }
711 } else {
712 bio_for_each_segment(bvec, bio, iter) {
713 bv_len = bvec.bv_len;
714 bv_off = bvec.bv_offset;
715
716 while (bv_len > 0) {
717 tgt = rsxx_get_dma_tgt(card, addr8);
718 laddr = rsxx_addr8_to_laddr(addr8, card);
719 dma_off = addr8 & RSXX_HW_BLK_MASK;
720 dma_len = min(bv_len,
721 RSXX_HW_BLK_SIZE - dma_off);
722
723 st = rsxx_queue_dma(card, &dma_list[tgt],
724 bio_data_dir(bio),
725 dma_off, dma_len,
726 laddr, bvec.bv_page,
727 bv_off, cb, cb_data);
728 if (st)
729 goto bvec_err;
730
731 dma_cnt[tgt]++;
732 atomic_inc(n_dmas);
733 addr8 += dma_len;
734 bv_off += dma_len;
735 bv_len -= dma_len;
736 }
737 }
738 }
739
740 for (i = 0; i < card->n_targets; i++) {
741 if (!list_empty(&dma_list[i])) {
742 spin_lock_bh(&card->ctrl[i].queue_lock);
743 card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
744 list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
745 spin_unlock_bh(&card->ctrl[i].queue_lock);
746
747 queue_work(card->ctrl[i].issue_wq,
748 &card->ctrl[i].issue_dma_work);
749 }
750 }
751
752 return 0;
753
754 bvec_err:
755 for (i = 0; i < card->n_targets; i++)
756 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
757 FREE_DMA);
758 return st;
759 }
760
761
762 /*----------------- DMA Engine Initialization & Setup -------------------*/
rsxx_hw_buffers_init(struct pci_dev * dev,struct rsxx_dma_ctrl * ctrl)763 int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
764 {
765 ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8,
766 &ctrl->status.dma_addr, GFP_KERNEL);
767 ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8,
768 &ctrl->cmd.dma_addr, GFP_KERNEL);
769 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
770 return -ENOMEM;
771
772 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
773 iowrite32(lower_32_bits(ctrl->status.dma_addr),
774 ctrl->regmap + SB_ADD_LO);
775 iowrite32(upper_32_bits(ctrl->status.dma_addr),
776 ctrl->regmap + SB_ADD_HI);
777
778 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
779 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
780 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
781
782 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
783 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
784 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
785 ctrl->status.idx);
786 return -EINVAL;
787 }
788 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
789 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
790
791 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
792 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
793 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
794 ctrl->status.idx);
795 return -EINVAL;
796 }
797 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
798 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
799
800 return 0;
801 }
802
rsxx_dma_ctrl_init(struct pci_dev * dev,struct rsxx_dma_ctrl * ctrl)803 static int rsxx_dma_ctrl_init(struct pci_dev *dev,
804 struct rsxx_dma_ctrl *ctrl)
805 {
806 int i;
807 int st;
808
809 memset(&ctrl->stats, 0, sizeof(ctrl->stats));
810
811 ctrl->trackers = vmalloc(DMA_TRACKER_LIST_SIZE8);
812 if (!ctrl->trackers)
813 return -ENOMEM;
814
815 ctrl->trackers->head = 0;
816 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
817 ctrl->trackers->list[i].next_tag = i + 1;
818 ctrl->trackers->list[i].dma = NULL;
819 }
820 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
821 spin_lock_init(&ctrl->trackers->lock);
822
823 spin_lock_init(&ctrl->queue_lock);
824 mutex_init(&ctrl->work_lock);
825 INIT_LIST_HEAD(&ctrl->queue);
826
827 timer_setup(&ctrl->activity_timer, dma_engine_stalled, 0);
828
829 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
830 if (!ctrl->issue_wq)
831 return -ENOMEM;
832
833 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
834 if (!ctrl->done_wq)
835 return -ENOMEM;
836
837 INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
838 INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
839
840 st = rsxx_hw_buffers_init(dev, ctrl);
841 if (st)
842 return st;
843
844 return 0;
845 }
846
rsxx_dma_stripe_setup(struct rsxx_cardinfo * card,unsigned int stripe_size8)847 static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
848 unsigned int stripe_size8)
849 {
850 if (!is_power_of_2(stripe_size8)) {
851 dev_err(CARD_TO_DEV(card),
852 "stripe_size is NOT a power of 2!\n");
853 return -EINVAL;
854 }
855
856 card->_stripe.lower_mask = stripe_size8 - 1;
857
858 card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
859 card->_stripe.upper_shift = ffs(card->n_targets) - 1;
860
861 card->_stripe.target_mask = card->n_targets - 1;
862 card->_stripe.target_shift = ffs(stripe_size8) - 1;
863
864 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
865 card->_stripe.lower_mask);
866 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
867 card->_stripe.upper_shift);
868 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
869 card->_stripe.upper_mask);
870 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
871 card->_stripe.target_mask);
872 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
873 card->_stripe.target_shift);
874
875 return 0;
876 }
877
rsxx_dma_configure(struct rsxx_cardinfo * card)878 int rsxx_dma_configure(struct rsxx_cardinfo *card)
879 {
880 u32 intr_coal;
881
882 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
883 card->config.data.intr_coal.count,
884 card->config.data.intr_coal.latency);
885 iowrite32(intr_coal, card->regmap + INTR_COAL);
886
887 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
888 }
889
rsxx_dma_setup(struct rsxx_cardinfo * card)890 int rsxx_dma_setup(struct rsxx_cardinfo *card)
891 {
892 unsigned long flags;
893 int st;
894 int i;
895
896 dev_info(CARD_TO_DEV(card),
897 "Initializing %d DMA targets\n",
898 card->n_targets);
899
900 /* Regmap is divided up into 4K chunks. One for each DMA channel */
901 for (i = 0; i < card->n_targets; i++)
902 card->ctrl[i].regmap = card->regmap + (i * 4096);
903
904 card->dma_fault = 0;
905
906 /* Reset the DMA queues */
907 rsxx_dma_queue_reset(card);
908
909 /************* Setup DMA Control *************/
910 for (i = 0; i < card->n_targets; i++) {
911 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
912 if (st)
913 goto failed_dma_setup;
914
915 card->ctrl[i].card = card;
916 card->ctrl[i].id = i;
917 }
918
919 card->scrub_hard = 1;
920
921 if (card->config_valid)
922 rsxx_dma_configure(card);
923
924 /* Enable the interrupts after all setup has completed. */
925 for (i = 0; i < card->n_targets; i++) {
926 spin_lock_irqsave(&card->irq_lock, flags);
927 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
928 spin_unlock_irqrestore(&card->irq_lock, flags);
929 }
930
931 return 0;
932
933 failed_dma_setup:
934 for (i = 0; i < card->n_targets; i++) {
935 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
936
937 if (ctrl->issue_wq) {
938 destroy_workqueue(ctrl->issue_wq);
939 ctrl->issue_wq = NULL;
940 }
941
942 if (ctrl->done_wq) {
943 destroy_workqueue(ctrl->done_wq);
944 ctrl->done_wq = NULL;
945 }
946
947 if (ctrl->trackers)
948 vfree(ctrl->trackers);
949
950 if (ctrl->status.buf)
951 dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
952 ctrl->status.buf,
953 ctrl->status.dma_addr);
954 if (ctrl->cmd.buf)
955 dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
956 ctrl->cmd.buf, ctrl->cmd.dma_addr);
957 }
958
959 return st;
960 }
961
rsxx_dma_cancel(struct rsxx_dma_ctrl * ctrl)962 int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
963 {
964 struct rsxx_dma *dma;
965 int i;
966 int cnt = 0;
967
968 /* Clean up issued DMAs */
969 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
970 dma = get_tracker_dma(ctrl->trackers, i);
971 if (dma) {
972 atomic_dec(&ctrl->stats.hw_q_depth);
973 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
974 push_tracker(ctrl->trackers, i);
975 cnt++;
976 }
977 }
978
979 return cnt;
980 }
981
rsxx_dma_destroy(struct rsxx_cardinfo * card)982 void rsxx_dma_destroy(struct rsxx_cardinfo *card)
983 {
984 struct rsxx_dma_ctrl *ctrl;
985 int i;
986
987 for (i = 0; i < card->n_targets; i++) {
988 ctrl = &card->ctrl[i];
989
990 if (ctrl->issue_wq) {
991 destroy_workqueue(ctrl->issue_wq);
992 ctrl->issue_wq = NULL;
993 }
994
995 if (ctrl->done_wq) {
996 destroy_workqueue(ctrl->done_wq);
997 ctrl->done_wq = NULL;
998 }
999
1000 if (timer_pending(&ctrl->activity_timer))
1001 del_timer_sync(&ctrl->activity_timer);
1002
1003 /* Clean up the DMA queue */
1004 spin_lock_bh(&ctrl->queue_lock);
1005 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
1006 spin_unlock_bh(&ctrl->queue_lock);
1007
1008 rsxx_dma_cancel(ctrl);
1009
1010 vfree(ctrl->trackers);
1011
1012 dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
1013 ctrl->status.buf, ctrl->status.dma_addr);
1014 dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
1015 ctrl->cmd.buf, ctrl->cmd.dma_addr);
1016 }
1017 }
1018
rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo * card)1019 int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1020 {
1021 int i;
1022 int j;
1023 int cnt;
1024 struct rsxx_dma *dma;
1025 struct list_head *issued_dmas;
1026
1027 issued_dmas = kcalloc(card->n_targets, sizeof(*issued_dmas),
1028 GFP_KERNEL);
1029 if (!issued_dmas)
1030 return -ENOMEM;
1031
1032 for (i = 0; i < card->n_targets; i++) {
1033 INIT_LIST_HEAD(&issued_dmas[i]);
1034 cnt = 0;
1035 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1036 dma = get_tracker_dma(card->ctrl[i].trackers, j);
1037 if (dma == NULL)
1038 continue;
1039
1040 if (dma->cmd == HW_CMD_BLK_WRITE)
1041 card->ctrl[i].stats.writes_issued--;
1042 else if (dma->cmd == HW_CMD_BLK_DISCARD)
1043 card->ctrl[i].stats.discards_issued--;
1044 else
1045 card->ctrl[i].stats.reads_issued--;
1046
1047 if (dma->cmd != HW_CMD_BLK_DISCARD) {
1048 dma_unmap_page(&card->dev->dev, dma->dma_addr,
1049 get_dma_size(dma),
1050 dma->cmd == HW_CMD_BLK_WRITE ?
1051 DMA_TO_DEVICE :
1052 DMA_FROM_DEVICE);
1053 }
1054
1055 list_add_tail(&dma->list, &issued_dmas[i]);
1056 push_tracker(card->ctrl[i].trackers, j);
1057 cnt++;
1058 }
1059
1060 spin_lock_bh(&card->ctrl[i].queue_lock);
1061 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1062
1063 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1064 card->ctrl[i].stats.sw_q_depth += cnt;
1065 card->ctrl[i].e_cnt = 0;
1066 spin_unlock_bh(&card->ctrl[i].queue_lock);
1067 }
1068
1069 kfree(issued_dmas);
1070
1071 return 0;
1072 }
1073
rsxx_dma_init(void)1074 int rsxx_dma_init(void)
1075 {
1076 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1077 if (!rsxx_dma_pool)
1078 return -ENOMEM;
1079
1080 return 0;
1081 }
1082
1083
rsxx_dma_cleanup(void)1084 void rsxx_dma_cleanup(void)
1085 {
1086 kmem_cache_destroy(rsxx_dma_pool);
1087 }
1088
1089
