1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Filename: dma.c
4 *
5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6 * Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7 *
8 * (C) Copyright 2013 IBM Corporation
9 */
10
11 #include <linux/slab.h>
12 #include "rsxx_priv.h"
13
14 struct rsxx_dma {
15 struct list_head list;
16 u8 cmd;
17 unsigned int laddr; /* Logical address */
18 struct {
19 u32 off;
20 u32 cnt;
21 } sub_page;
22 dma_addr_t dma_addr;
23 struct page *page;
24 unsigned int pg_off; /* Page Offset */
25 rsxx_dma_cb cb;
26 void *cb_data;
27 };
28
29 /* This timeout is used to detect a stalled DMA channel */
30 #define DMA_ACTIVITY_TIMEOUT msecs_to_jiffies(10000)
31
32 struct hw_status {
33 u8 status;
34 u8 tag;
35 __le16 count;
36 __le32 _rsvd2;
37 __le64 _rsvd3;
38 } __packed;
39
40 enum rsxx_dma_status {
41 DMA_SW_ERR = 0x1,
42 DMA_HW_FAULT = 0x2,
43 DMA_CANCELLED = 0x4,
44 };
45
46 struct hw_cmd {
47 u8 command;
48 u8 tag;
49 u8 _rsvd;
50 u8 sub_page; /* Bit[0:2]: 512byte offset */
51 /* Bit[4:6]: 512byte count */
52 __le32 device_addr;
53 __le64 host_addr;
54 } __packed;
55
56 enum rsxx_hw_cmd {
57 HW_CMD_BLK_DISCARD = 0x70,
58 HW_CMD_BLK_WRITE = 0x80,
59 HW_CMD_BLK_READ = 0xC0,
60 HW_CMD_BLK_RECON_READ = 0xE0,
61 };
62
63 enum rsxx_hw_status {
64 HW_STATUS_CRC = 0x01,
65 HW_STATUS_HARD_ERR = 0x02,
66 HW_STATUS_SOFT_ERR = 0x04,
67 HW_STATUS_FAULT = 0x08,
68 };
69
70 static struct kmem_cache *rsxx_dma_pool;
71
72 struct dma_tracker {
73 int next_tag;
74 struct rsxx_dma *dma;
75 };
76
77 struct dma_tracker_list {
78 spinlock_t lock;
79 int head;
80 struct dma_tracker list[];
81 };
82
83
84 /*----------------- Misc Utility Functions -------------------*/
rsxx_addr8_to_laddr(u64 addr8,struct rsxx_cardinfo * card)85 static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
86 {
87 unsigned long long tgt_addr8;
88
89 tgt_addr8 = ((addr8 >> card->_stripe.upper_shift) &
90 card->_stripe.upper_mask) |
91 ((addr8) & card->_stripe.lower_mask);
92 do_div(tgt_addr8, RSXX_HW_BLK_SIZE);
93 return tgt_addr8;
94 }
95
rsxx_get_dma_tgt(struct rsxx_cardinfo * card,u64 addr8)96 static unsigned int rsxx_get_dma_tgt(struct rsxx_cardinfo *card, u64 addr8)
97 {
98 unsigned int tgt;
99
100 tgt = (addr8 >> card->_stripe.target_shift) & card->_stripe.target_mask;
101
102 return tgt;
103 }
104
rsxx_dma_queue_reset(struct rsxx_cardinfo * card)105 void rsxx_dma_queue_reset(struct rsxx_cardinfo *card)
106 {
107 /* Reset all DMA Command/Status Queues */
108 iowrite32(DMA_QUEUE_RESET, card->regmap + RESET);
109 }
110
get_dma_size(struct rsxx_dma * dma)111 static unsigned int get_dma_size(struct rsxx_dma *dma)
112 {
113 if (dma->sub_page.cnt)
114 return dma->sub_page.cnt << 9;
115 else
116 return RSXX_HW_BLK_SIZE;
117 }
118
119
120 /*----------------- DMA Tracker -------------------*/
set_tracker_dma(struct dma_tracker_list * trackers,int tag,struct rsxx_dma * dma)121 static void set_tracker_dma(struct dma_tracker_list *trackers,
122 int tag,
123 struct rsxx_dma *dma)
124 {
125 trackers->list[tag].dma = dma;
126 }
127
get_tracker_dma(struct dma_tracker_list * trackers,int tag)128 static struct rsxx_dma *get_tracker_dma(struct dma_tracker_list *trackers,
129 int tag)
130 {
131 return trackers->list[tag].dma;
132 }
133
pop_tracker(struct dma_tracker_list * trackers)134 static int pop_tracker(struct dma_tracker_list *trackers)
135 {
136 int tag;
137
138 spin_lock(&trackers->lock);
139 tag = trackers->head;
140 if (tag != -1) {
141 trackers->head = trackers->list[tag].next_tag;
142 trackers->list[tag].next_tag = -1;
143 }
144 spin_unlock(&trackers->lock);
145
146 return tag;
147 }
148
push_tracker(struct dma_tracker_list * trackers,int tag)149 static void push_tracker(struct dma_tracker_list *trackers, int tag)
150 {
151 spin_lock(&trackers->lock);
152 trackers->list[tag].next_tag = trackers->head;
153 trackers->head = tag;
154 trackers->list[tag].dma = NULL;
155 spin_unlock(&trackers->lock);
156 }
157
158
159 /*----------------- Interrupt Coalescing -------------*/
160 /*
161 * Interrupt Coalescing Register Format:
162 * Interrupt Timer (64ns units) [15:0]
163 * Interrupt Count [24:16]
164 * Reserved [31:25]
165 */
166 #define INTR_COAL_LATENCY_MASK (0x0000ffff)
167
168 #define INTR_COAL_COUNT_SHIFT 16
169 #define INTR_COAL_COUNT_BITS 9
170 #define INTR_COAL_COUNT_MASK (((1 << INTR_COAL_COUNT_BITS) - 1) << \
171 INTR_COAL_COUNT_SHIFT)
172 #define INTR_COAL_LATENCY_UNITS_NS 64
173
174
dma_intr_coal_val(u32 mode,u32 count,u32 latency)175 static u32 dma_intr_coal_val(u32 mode, u32 count, u32 latency)
176 {
177 u32 latency_units = latency / INTR_COAL_LATENCY_UNITS_NS;
178
179 if (mode == RSXX_INTR_COAL_DISABLED)
180 return 0;
181
182 return ((count << INTR_COAL_COUNT_SHIFT) & INTR_COAL_COUNT_MASK) |
183 (latency_units & INTR_COAL_LATENCY_MASK);
184
185 }
186
dma_intr_coal_auto_tune(struct rsxx_cardinfo * card)187 static void dma_intr_coal_auto_tune(struct rsxx_cardinfo *card)
188 {
189 int i;
190 u32 q_depth = 0;
191 u32 intr_coal;
192
193 if (card->config.data.intr_coal.mode != RSXX_INTR_COAL_AUTO_TUNE ||
194 unlikely(card->eeh_state))
195 return;
196
197 for (i = 0; i < card->n_targets; i++)
198 q_depth += atomic_read(&card->ctrl[i].stats.hw_q_depth);
199
200 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
201 q_depth / 2,
202 card->config.data.intr_coal.latency);
203 iowrite32(intr_coal, card->regmap + INTR_COAL);
204 }
205
206 /*----------------- RSXX DMA Handling -------------------*/
rsxx_free_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma)207 static void rsxx_free_dma(struct rsxx_dma_ctrl *ctrl, struct rsxx_dma *dma)
208 {
209 if (dma->cmd != HW_CMD_BLK_DISCARD) {
210 if (!dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
211 dma_unmap_page(&ctrl->card->dev->dev, dma->dma_addr,
212 get_dma_size(dma),
213 dma->cmd == HW_CMD_BLK_WRITE ?
214 DMA_TO_DEVICE :
215 DMA_FROM_DEVICE);
216 }
217 }
218
219 kmem_cache_free(rsxx_dma_pool, dma);
220 }
221
rsxx_complete_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma,unsigned int status)222 static void rsxx_complete_dma(struct rsxx_dma_ctrl *ctrl,
223 struct rsxx_dma *dma,
224 unsigned int status)
225 {
226 if (status & DMA_SW_ERR)
227 ctrl->stats.dma_sw_err++;
228 if (status & DMA_HW_FAULT)
229 ctrl->stats.dma_hw_fault++;
230 if (status & DMA_CANCELLED)
231 ctrl->stats.dma_cancelled++;
232
233 if (dma->cb)
234 dma->cb(ctrl->card, dma->cb_data, status ? 1 : 0);
235
236 rsxx_free_dma(ctrl, dma);
237 }
238
rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl * ctrl,struct list_head * q,unsigned int done)239 int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
240 struct list_head *q, unsigned int done)
241 {
242 struct rsxx_dma *dma;
243 struct rsxx_dma *tmp;
244 int cnt = 0;
245
246 list_for_each_entry_safe(dma, tmp, q, list) {
247 list_del(&dma->list);
248 if (done & COMPLETE_DMA)
249 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
250 else
251 rsxx_free_dma(ctrl, dma);
252 cnt++;
253 }
254
255 return cnt;
256 }
257
rsxx_requeue_dma(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma)258 static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
259 struct rsxx_dma *dma)
260 {
261 /*
262 * Requeued DMAs go to the front of the queue so they are issued
263 * first.
264 */
265 spin_lock_bh(&ctrl->queue_lock);
266 ctrl->stats.sw_q_depth++;
267 list_add(&dma->list, &ctrl->queue);
268 spin_unlock_bh(&ctrl->queue_lock);
269 }
270
rsxx_handle_dma_error(struct rsxx_dma_ctrl * ctrl,struct rsxx_dma * dma,u8 hw_st)271 static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
272 struct rsxx_dma *dma,
273 u8 hw_st)
274 {
275 unsigned int status = 0;
276 int requeue_cmd = 0;
277
278 dev_dbg(CARD_TO_DEV(ctrl->card),
279 "Handling DMA error(cmd x%02x, laddr x%08x st:x%02x)\n",
280 dma->cmd, dma->laddr, hw_st);
281
282 if (hw_st & HW_STATUS_CRC)
283 ctrl->stats.crc_errors++;
284 if (hw_st & HW_STATUS_HARD_ERR)
285 ctrl->stats.hard_errors++;
286 if (hw_st & HW_STATUS_SOFT_ERR)
287 ctrl->stats.soft_errors++;
288
289 switch (dma->cmd) {
290 case HW_CMD_BLK_READ:
291 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
292 if (ctrl->card->scrub_hard) {
293 dma->cmd = HW_CMD_BLK_RECON_READ;
294 requeue_cmd = 1;
295 ctrl->stats.reads_retried++;
296 } else {
297 status |= DMA_HW_FAULT;
298 ctrl->stats.reads_failed++;
299 }
300 } else if (hw_st & HW_STATUS_FAULT) {
301 status |= DMA_HW_FAULT;
302 ctrl->stats.reads_failed++;
303 }
304
305 break;
306 case HW_CMD_BLK_RECON_READ:
307 if (hw_st & (HW_STATUS_CRC | HW_STATUS_HARD_ERR)) {
308 /* Data could not be reconstructed. */
309 status |= DMA_HW_FAULT;
310 ctrl->stats.reads_failed++;
311 }
312
313 break;
314 case HW_CMD_BLK_WRITE:
315 status |= DMA_HW_FAULT;
316 ctrl->stats.writes_failed++;
317
318 break;
319 case HW_CMD_BLK_DISCARD:
320 status |= DMA_HW_FAULT;
321 ctrl->stats.discards_failed++;
322
323 break;
324 default:
325 dev_err(CARD_TO_DEV(ctrl->card),
326 "Unknown command in DMA!(cmd: x%02x "
327 "laddr x%08x st: x%02x\n",
328 dma->cmd, dma->laddr, hw_st);
329 status |= DMA_SW_ERR;
330
331 break;
332 }
333
334 if (requeue_cmd)
335 rsxx_requeue_dma(ctrl, dma);
336 else
337 rsxx_complete_dma(ctrl, dma, status);
338 }
339
dma_engine_stalled(struct timer_list * t)340 static void dma_engine_stalled(struct timer_list *t)
341 {
342 struct rsxx_dma_ctrl *ctrl = from_timer(ctrl, t, activity_timer);
343 int cnt;
344
345 if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
346 unlikely(ctrl->card->eeh_state))
347 return;
348
349 if (ctrl->cmd.idx != ioread32(ctrl->regmap + SW_CMD_IDX)) {
350 /*
351 * The dma engine was stalled because the SW_CMD_IDX write
352 * was lost. Issue it again to recover.
353 */
354 dev_warn(CARD_TO_DEV(ctrl->card),
355 "SW_CMD_IDX write was lost, re-writing...\n");
356 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
357 mod_timer(&ctrl->activity_timer,
358 jiffies + DMA_ACTIVITY_TIMEOUT);
359 } else {
360 dev_warn(CARD_TO_DEV(ctrl->card),
361 "DMA channel %d has stalled, faulting interface.\n",
362 ctrl->id);
363 ctrl->card->dma_fault = 1;
364
365 /* Clean up the DMA queue */
366 spin_lock(&ctrl->queue_lock);
367 cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
368 spin_unlock(&ctrl->queue_lock);
369
370 cnt += rsxx_dma_cancel(ctrl);
371
372 if (cnt)
373 dev_info(CARD_TO_DEV(ctrl->card),
374 "Freed %d queued DMAs on channel %d\n",
375 cnt, ctrl->id);
376 }
377 }
378
rsxx_issue_dmas(struct rsxx_dma_ctrl * ctrl)379 static void rsxx_issue_dmas(struct rsxx_dma_ctrl *ctrl)
380 {
381 struct rsxx_dma *dma;
382 int tag;
383 int cmds_pending = 0;
384 struct hw_cmd *hw_cmd_buf;
385 int dir;
386
387 hw_cmd_buf = ctrl->cmd.buf;
388
389 if (unlikely(ctrl->card->halt) ||
390 unlikely(ctrl->card->eeh_state))
391 return;
392
393 while (1) {
394 spin_lock_bh(&ctrl->queue_lock);
395 if (list_empty(&ctrl->queue)) {
396 spin_unlock_bh(&ctrl->queue_lock);
397 break;
398 }
399 spin_unlock_bh(&ctrl->queue_lock);
400
401 tag = pop_tracker(ctrl->trackers);
402 if (tag == -1)
403 break;
404
405 spin_lock_bh(&ctrl->queue_lock);
406 dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
407 list_del(&dma->list);
408 ctrl->stats.sw_q_depth--;
409 spin_unlock_bh(&ctrl->queue_lock);
410
411 /*
412 * This will catch any DMAs that slipped in right before the
413 * fault, but was queued after all the other DMAs were
414 * cancelled.
415 */
416 if (unlikely(ctrl->card->dma_fault)) {
417 push_tracker(ctrl->trackers, tag);
418 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
419 continue;
420 }
421
422 if (dma->cmd != HW_CMD_BLK_DISCARD) {
423 if (dma->cmd == HW_CMD_BLK_WRITE)
424 dir = DMA_TO_DEVICE;
425 else
426 dir = DMA_FROM_DEVICE;
427
428 /*
429 * The function dma_map_page is placed here because we
430 * can only, by design, issue up to 255 commands to the
431 * hardware at one time per DMA channel. So the maximum
432 * amount of mapped memory would be 255 * 4 channels *
433 * 4096 Bytes which is less than 2GB, the limit of a x8
434 * Non-HWWD PCIe slot. This way the dma_map_page
435 * function should never fail because of a lack of
436 * mappable memory.
437 */
438 dma->dma_addr = dma_map_page(&ctrl->card->dev->dev, dma->page,
439 dma->pg_off, dma->sub_page.cnt << 9, dir);
440 if (dma_mapping_error(&ctrl->card->dev->dev, dma->dma_addr)) {
441 push_tracker(ctrl->trackers, tag);
442 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
443 continue;
444 }
445 }
446
447 set_tracker_dma(ctrl->trackers, tag, dma);
448 hw_cmd_buf[ctrl->cmd.idx].command = dma->cmd;
449 hw_cmd_buf[ctrl->cmd.idx].tag = tag;
450 hw_cmd_buf[ctrl->cmd.idx]._rsvd = 0;
451 hw_cmd_buf[ctrl->cmd.idx].sub_page =
452 ((dma->sub_page.cnt & 0x7) << 4) |
453 (dma->sub_page.off & 0x7);
454
455 hw_cmd_buf[ctrl->cmd.idx].device_addr =
456 cpu_to_le32(dma->laddr);
457
458 hw_cmd_buf[ctrl->cmd.idx].host_addr =
459 cpu_to_le64(dma->dma_addr);
460
461 dev_dbg(CARD_TO_DEV(ctrl->card),
462 "Issue DMA%d(laddr %d tag %d) to idx %d\n",
463 ctrl->id, dma->laddr, tag, ctrl->cmd.idx);
464
465 ctrl->cmd.idx = (ctrl->cmd.idx + 1) & RSXX_CS_IDX_MASK;
466 cmds_pending++;
467
468 if (dma->cmd == HW_CMD_BLK_WRITE)
469 ctrl->stats.writes_issued++;
470 else if (dma->cmd == HW_CMD_BLK_DISCARD)
471 ctrl->stats.discards_issued++;
472 else
473 ctrl->stats.reads_issued++;
474 }
475
476 /* Let HW know we've queued commands. */
477 if (cmds_pending) {
478 atomic_add(cmds_pending, &ctrl->stats.hw_q_depth);
479 mod_timer(&ctrl->activity_timer,
480 jiffies + DMA_ACTIVITY_TIMEOUT);
481
482 if (unlikely(ctrl->card->eeh_state)) {
483 del_timer_sync(&ctrl->activity_timer);
484 return;
485 }
486
487 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
488 }
489 }
490
rsxx_dma_done(struct rsxx_dma_ctrl * ctrl)491 static void rsxx_dma_done(struct rsxx_dma_ctrl *ctrl)
492 {
493 struct rsxx_dma *dma;
494 unsigned long flags;
495 u16 count;
496 u8 status;
497 u8 tag;
498 struct hw_status *hw_st_buf;
499
500 hw_st_buf = ctrl->status.buf;
501
502 if (unlikely(ctrl->card->halt) ||
503 unlikely(ctrl->card->dma_fault) ||
504 unlikely(ctrl->card->eeh_state))
505 return;
506
507 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
508
509 while (count == ctrl->e_cnt) {
510 /*
511 * The read memory-barrier is necessary to keep aggressive
512 * processors/optimizers (such as the PPC Apple G5) from
513 * reordering the following status-buffer tag & status read
514 * *before* the count read on subsequent iterations of the
515 * loop!
516 */
517 rmb();
518
519 status = hw_st_buf[ctrl->status.idx].status;
520 tag = hw_st_buf[ctrl->status.idx].tag;
521
522 dma = get_tracker_dma(ctrl->trackers, tag);
523 if (dma == NULL) {
524 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
525 rsxx_disable_ier(ctrl->card, CR_INTR_DMA_ALL);
526 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
527
528 dev_err(CARD_TO_DEV(ctrl->card),
529 "No tracker for tag %d "
530 "(idx %d id %d)\n",
531 tag, ctrl->status.idx, ctrl->id);
532 return;
533 }
534
535 dev_dbg(CARD_TO_DEV(ctrl->card),
536 "Completing DMA%d"
537 "(laddr x%x tag %d st: x%x cnt: x%04x) from idx %d.\n",
538 ctrl->id, dma->laddr, tag, status, count,
539 ctrl->status.idx);
540
541 atomic_dec(&ctrl->stats.hw_q_depth);
542
543 mod_timer(&ctrl->activity_timer,
544 jiffies + DMA_ACTIVITY_TIMEOUT);
545
546 if (status)
547 rsxx_handle_dma_error(ctrl, dma, status);
548 else
549 rsxx_complete_dma(ctrl, dma, 0);
550
551 push_tracker(ctrl->trackers, tag);
552
553 ctrl->status.idx = (ctrl->status.idx + 1) &
554 RSXX_CS_IDX_MASK;
555 ctrl->e_cnt++;
556
557 count = le16_to_cpu(hw_st_buf[ctrl->status.idx].count);
558 }
559
560 dma_intr_coal_auto_tune(ctrl->card);
561
562 if (atomic_read(&ctrl->stats.hw_q_depth) == 0)
563 del_timer_sync(&ctrl->activity_timer);
564
565 spin_lock_irqsave(&ctrl->card->irq_lock, flags);
566 rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
567 spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);
568
569 spin_lock_bh(&ctrl->queue_lock);
570 if (ctrl->stats.sw_q_depth)
571 queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
572 spin_unlock_bh(&ctrl->queue_lock);
573 }
574
rsxx_schedule_issue(struct work_struct * work)575 static void rsxx_schedule_issue(struct work_struct *work)
576 {
577 struct rsxx_dma_ctrl *ctrl;
578
579 ctrl = container_of(work, struct rsxx_dma_ctrl, issue_dma_work);
580
581 mutex_lock(&ctrl->work_lock);
582 rsxx_issue_dmas(ctrl);
583 mutex_unlock(&ctrl->work_lock);
584 }
585
rsxx_schedule_done(struct work_struct * work)586 static void rsxx_schedule_done(struct work_struct *work)
587 {
588 struct rsxx_dma_ctrl *ctrl;
589
590 ctrl = container_of(work, struct rsxx_dma_ctrl, dma_done_work);
591
592 mutex_lock(&ctrl->work_lock);
593 rsxx_dma_done(ctrl);
594 mutex_unlock(&ctrl->work_lock);
595 }
596
rsxx_queue_discard(struct rsxx_cardinfo * card,struct list_head * q,unsigned int laddr,rsxx_dma_cb cb,void * cb_data)597 static blk_status_t rsxx_queue_discard(struct rsxx_cardinfo *card,
598 struct list_head *q,
599 unsigned int laddr,
600 rsxx_dma_cb cb,
601 void *cb_data)
602 {
603 struct rsxx_dma *dma;
604
605 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
606 if (!dma)
607 return BLK_STS_RESOURCE;
608
609 dma->cmd = HW_CMD_BLK_DISCARD;
610 dma->laddr = laddr;
611 dma->dma_addr = 0;
612 dma->sub_page.off = 0;
613 dma->sub_page.cnt = 0;
614 dma->page = NULL;
615 dma->pg_off = 0;
616 dma->cb = cb;
617 dma->cb_data = cb_data;
618
619 dev_dbg(CARD_TO_DEV(card), "Queuing[D] laddr %x\n", dma->laddr);
620
621 list_add_tail(&dma->list, q);
622
623 return 0;
624 }
625
rsxx_queue_dma(struct rsxx_cardinfo * card,struct list_head * q,int dir,unsigned int dma_off,unsigned int dma_len,unsigned int laddr,struct page * page,unsigned int pg_off,rsxx_dma_cb cb,void * cb_data)626 static blk_status_t rsxx_queue_dma(struct rsxx_cardinfo *card,
627 struct list_head *q,
628 int dir,
629 unsigned int dma_off,
630 unsigned int dma_len,
631 unsigned int laddr,
632 struct page *page,
633 unsigned int pg_off,
634 rsxx_dma_cb cb,
635 void *cb_data)
636 {
637 struct rsxx_dma *dma;
638
639 dma = kmem_cache_alloc(rsxx_dma_pool, GFP_KERNEL);
640 if (!dma)
641 return BLK_STS_RESOURCE;
642
643 dma->cmd = dir ? HW_CMD_BLK_WRITE : HW_CMD_BLK_READ;
644 dma->laddr = laddr;
645 dma->sub_page.off = (dma_off >> 9);
646 dma->sub_page.cnt = (dma_len >> 9);
647 dma->page = page;
648 dma->pg_off = pg_off;
649 dma->cb = cb;
650 dma->cb_data = cb_data;
651
652 dev_dbg(CARD_TO_DEV(card),
653 "Queuing[%c] laddr %x off %d cnt %d page %p pg_off %d\n",
654 dir ? 'W' : 'R', dma->laddr, dma->sub_page.off,
655 dma->sub_page.cnt, dma->page, dma->pg_off);
656
657 /* Queue the DMA */
658 list_add_tail(&dma->list, q);
659
660 return 0;
661 }
662
rsxx_dma_queue_bio(struct rsxx_cardinfo * card,struct bio * bio,atomic_t * n_dmas,rsxx_dma_cb cb,void * cb_data)663 blk_status_t rsxx_dma_queue_bio(struct rsxx_cardinfo *card,
664 struct bio *bio,
665 atomic_t *n_dmas,
666 rsxx_dma_cb cb,
667 void *cb_data)
668 {
669 struct list_head dma_list[RSXX_MAX_TARGETS];
670 struct bio_vec bvec;
671 struct bvec_iter iter;
672 unsigned long long addr8;
673 unsigned int laddr;
674 unsigned int bv_len;
675 unsigned int bv_off;
676 unsigned int dma_off;
677 unsigned int dma_len;
678 int dma_cnt[RSXX_MAX_TARGETS];
679 int tgt;
680 blk_status_t st;
681 int i;
682
683 addr8 = bio->bi_iter.bi_sector << 9; /* sectors are 512 bytes */
684 atomic_set(n_dmas, 0);
685
686 for (i = 0; i < card->n_targets; i++) {
687 INIT_LIST_HEAD(&dma_list[i]);
688 dma_cnt[i] = 0;
689 }
690
691 if (bio_op(bio) == REQ_OP_DISCARD) {
692 bv_len = bio->bi_iter.bi_size;
693
694 while (bv_len > 0) {
695 tgt = rsxx_get_dma_tgt(card, addr8);
696 laddr = rsxx_addr8_to_laddr(addr8, card);
697
698 st = rsxx_queue_discard(card, &dma_list[tgt], laddr,
699 cb, cb_data);
700 if (st)
701 goto bvec_err;
702
703 dma_cnt[tgt]++;
704 atomic_inc(n_dmas);
705 addr8 += RSXX_HW_BLK_SIZE;
706 bv_len -= RSXX_HW_BLK_SIZE;
707 }
708 } else {
709 bio_for_each_segment(bvec, bio, iter) {
710 bv_len = bvec.bv_len;
711 bv_off = bvec.bv_offset;
712
713 while (bv_len > 0) {
714 tgt = rsxx_get_dma_tgt(card, addr8);
715 laddr = rsxx_addr8_to_laddr(addr8, card);
716 dma_off = addr8 & RSXX_HW_BLK_MASK;
717 dma_len = min(bv_len,
718 RSXX_HW_BLK_SIZE - dma_off);
719
720 st = rsxx_queue_dma(card, &dma_list[tgt],
721 bio_data_dir(bio),
722 dma_off, dma_len,
723 laddr, bvec.bv_page,
724 bv_off, cb, cb_data);
725 if (st)
726 goto bvec_err;
727
728 dma_cnt[tgt]++;
729 atomic_inc(n_dmas);
730 addr8 += dma_len;
731 bv_off += dma_len;
732 bv_len -= dma_len;
733 }
734 }
735 }
736
737 for (i = 0; i < card->n_targets; i++) {
738 if (!list_empty(&dma_list[i])) {
739 spin_lock_bh(&card->ctrl[i].queue_lock);
740 card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
741 list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
742 spin_unlock_bh(&card->ctrl[i].queue_lock);
743
744 queue_work(card->ctrl[i].issue_wq,
745 &card->ctrl[i].issue_dma_work);
746 }
747 }
748
749 return 0;
750
751 bvec_err:
752 for (i = 0; i < card->n_targets; i++)
753 rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i],
754 FREE_DMA);
755 return st;
756 }
757
758
759 /*----------------- DMA Engine Initialization & Setup -------------------*/
rsxx_hw_buffers_init(struct pci_dev * dev,struct rsxx_dma_ctrl * ctrl)760 int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl)
761 {
762 ctrl->status.buf = dma_alloc_coherent(&dev->dev, STATUS_BUFFER_SIZE8,
763 &ctrl->status.dma_addr, GFP_KERNEL);
764 ctrl->cmd.buf = dma_alloc_coherent(&dev->dev, COMMAND_BUFFER_SIZE8,
765 &ctrl->cmd.dma_addr, GFP_KERNEL);
766 if (ctrl->status.buf == NULL || ctrl->cmd.buf == NULL)
767 return -ENOMEM;
768
769 memset(ctrl->status.buf, 0xac, STATUS_BUFFER_SIZE8);
770 iowrite32(lower_32_bits(ctrl->status.dma_addr),
771 ctrl->regmap + SB_ADD_LO);
772 iowrite32(upper_32_bits(ctrl->status.dma_addr),
773 ctrl->regmap + SB_ADD_HI);
774
775 memset(ctrl->cmd.buf, 0x83, COMMAND_BUFFER_SIZE8);
776 iowrite32(lower_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_LO);
777 iowrite32(upper_32_bits(ctrl->cmd.dma_addr), ctrl->regmap + CB_ADD_HI);
778
779 ctrl->status.idx = ioread32(ctrl->regmap + HW_STATUS_CNT);
780 if (ctrl->status.idx > RSXX_MAX_OUTSTANDING_CMDS) {
781 dev_crit(&dev->dev, "Failed reading status cnt x%x\n",
782 ctrl->status.idx);
783 return -EINVAL;
784 }
785 iowrite32(ctrl->status.idx, ctrl->regmap + HW_STATUS_CNT);
786 iowrite32(ctrl->status.idx, ctrl->regmap + SW_STATUS_CNT);
787
788 ctrl->cmd.idx = ioread32(ctrl->regmap + HW_CMD_IDX);
789 if (ctrl->cmd.idx > RSXX_MAX_OUTSTANDING_CMDS) {
790 dev_crit(&dev->dev, "Failed reading cmd cnt x%x\n",
791 ctrl->status.idx);
792 return -EINVAL;
793 }
794 iowrite32(ctrl->cmd.idx, ctrl->regmap + HW_CMD_IDX);
795 iowrite32(ctrl->cmd.idx, ctrl->regmap + SW_CMD_IDX);
796
797 return 0;
798 }
799
rsxx_dma_ctrl_init(struct pci_dev * dev,struct rsxx_dma_ctrl * ctrl)800 static int rsxx_dma_ctrl_init(struct pci_dev *dev,
801 struct rsxx_dma_ctrl *ctrl)
802 {
803 int i;
804 int st;
805
806 memset(&ctrl->stats, 0, sizeof(ctrl->stats));
807
808 ctrl->trackers = vmalloc(struct_size(ctrl->trackers, list,
809 RSXX_MAX_OUTSTANDING_CMDS));
810 if (!ctrl->trackers)
811 return -ENOMEM;
812
813 ctrl->trackers->head = 0;
814 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
815 ctrl->trackers->list[i].next_tag = i + 1;
816 ctrl->trackers->list[i].dma = NULL;
817 }
818 ctrl->trackers->list[RSXX_MAX_OUTSTANDING_CMDS-1].next_tag = -1;
819 spin_lock_init(&ctrl->trackers->lock);
820
821 spin_lock_init(&ctrl->queue_lock);
822 mutex_init(&ctrl->work_lock);
823 INIT_LIST_HEAD(&ctrl->queue);
824
825 timer_setup(&ctrl->activity_timer, dma_engine_stalled, 0);
826
827 ctrl->issue_wq = alloc_ordered_workqueue(DRIVER_NAME"_issue", 0);
828 if (!ctrl->issue_wq)
829 return -ENOMEM;
830
831 ctrl->done_wq = alloc_ordered_workqueue(DRIVER_NAME"_done", 0);
832 if (!ctrl->done_wq)
833 return -ENOMEM;
834
835 INIT_WORK(&ctrl->issue_dma_work, rsxx_schedule_issue);
836 INIT_WORK(&ctrl->dma_done_work, rsxx_schedule_done);
837
838 st = rsxx_hw_buffers_init(dev, ctrl);
839 if (st)
840 return st;
841
842 return 0;
843 }
844
rsxx_dma_stripe_setup(struct rsxx_cardinfo * card,unsigned int stripe_size8)845 static int rsxx_dma_stripe_setup(struct rsxx_cardinfo *card,
846 unsigned int stripe_size8)
847 {
848 if (!is_power_of_2(stripe_size8)) {
849 dev_err(CARD_TO_DEV(card),
850 "stripe_size is NOT a power of 2!\n");
851 return -EINVAL;
852 }
853
854 card->_stripe.lower_mask = stripe_size8 - 1;
855
856 card->_stripe.upper_mask = ~(card->_stripe.lower_mask);
857 card->_stripe.upper_shift = ffs(card->n_targets) - 1;
858
859 card->_stripe.target_mask = card->n_targets - 1;
860 card->_stripe.target_shift = ffs(stripe_size8) - 1;
861
862 dev_dbg(CARD_TO_DEV(card), "_stripe.lower_mask = x%016llx\n",
863 card->_stripe.lower_mask);
864 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_shift = x%016llx\n",
865 card->_stripe.upper_shift);
866 dev_dbg(CARD_TO_DEV(card), "_stripe.upper_mask = x%016llx\n",
867 card->_stripe.upper_mask);
868 dev_dbg(CARD_TO_DEV(card), "_stripe.target_mask = x%016llx\n",
869 card->_stripe.target_mask);
870 dev_dbg(CARD_TO_DEV(card), "_stripe.target_shift = x%016llx\n",
871 card->_stripe.target_shift);
872
873 return 0;
874 }
875
rsxx_dma_configure(struct rsxx_cardinfo * card)876 int rsxx_dma_configure(struct rsxx_cardinfo *card)
877 {
878 u32 intr_coal;
879
880 intr_coal = dma_intr_coal_val(card->config.data.intr_coal.mode,
881 card->config.data.intr_coal.count,
882 card->config.data.intr_coal.latency);
883 iowrite32(intr_coal, card->regmap + INTR_COAL);
884
885 return rsxx_dma_stripe_setup(card, card->config.data.stripe_size);
886 }
887
rsxx_dma_setup(struct rsxx_cardinfo * card)888 int rsxx_dma_setup(struct rsxx_cardinfo *card)
889 {
890 unsigned long flags;
891 int st;
892 int i;
893
894 dev_info(CARD_TO_DEV(card),
895 "Initializing %d DMA targets\n",
896 card->n_targets);
897
898 /* Regmap is divided up into 4K chunks. One for each DMA channel */
899 for (i = 0; i < card->n_targets; i++)
900 card->ctrl[i].regmap = card->regmap + (i * 4096);
901
902 card->dma_fault = 0;
903
904 /* Reset the DMA queues */
905 rsxx_dma_queue_reset(card);
906
907 /************* Setup DMA Control *************/
908 for (i = 0; i < card->n_targets; i++) {
909 st = rsxx_dma_ctrl_init(card->dev, &card->ctrl[i]);
910 if (st)
911 goto failed_dma_setup;
912
913 card->ctrl[i].card = card;
914 card->ctrl[i].id = i;
915 }
916
917 card->scrub_hard = 1;
918
919 if (card->config_valid)
920 rsxx_dma_configure(card);
921
922 /* Enable the interrupts after all setup has completed. */
923 for (i = 0; i < card->n_targets; i++) {
924 spin_lock_irqsave(&card->irq_lock, flags);
925 rsxx_enable_ier_and_isr(card, CR_INTR_DMA(i));
926 spin_unlock_irqrestore(&card->irq_lock, flags);
927 }
928
929 return 0;
930
931 failed_dma_setup:
932 for (i = 0; i < card->n_targets; i++) {
933 struct rsxx_dma_ctrl *ctrl = &card->ctrl[i];
934
935 if (ctrl->issue_wq) {
936 destroy_workqueue(ctrl->issue_wq);
937 ctrl->issue_wq = NULL;
938 }
939
940 if (ctrl->done_wq) {
941 destroy_workqueue(ctrl->done_wq);
942 ctrl->done_wq = NULL;
943 }
944
945 vfree(ctrl->trackers);
946
947 if (ctrl->status.buf)
948 dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
949 ctrl->status.buf,
950 ctrl->status.dma_addr);
951 if (ctrl->cmd.buf)
952 dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
953 ctrl->cmd.buf, ctrl->cmd.dma_addr);
954 }
955
956 return st;
957 }
958
rsxx_dma_cancel(struct rsxx_dma_ctrl * ctrl)959 int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
960 {
961 struct rsxx_dma *dma;
962 int i;
963 int cnt = 0;
964
965 /* Clean up issued DMAs */
966 for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
967 dma = get_tracker_dma(ctrl->trackers, i);
968 if (dma) {
969 atomic_dec(&ctrl->stats.hw_q_depth);
970 rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
971 push_tracker(ctrl->trackers, i);
972 cnt++;
973 }
974 }
975
976 return cnt;
977 }
978
rsxx_dma_destroy(struct rsxx_cardinfo * card)979 void rsxx_dma_destroy(struct rsxx_cardinfo *card)
980 {
981 struct rsxx_dma_ctrl *ctrl;
982 int i;
983
984 for (i = 0; i < card->n_targets; i++) {
985 ctrl = &card->ctrl[i];
986
987 if (ctrl->issue_wq) {
988 destroy_workqueue(ctrl->issue_wq);
989 ctrl->issue_wq = NULL;
990 }
991
992 if (ctrl->done_wq) {
993 destroy_workqueue(ctrl->done_wq);
994 ctrl->done_wq = NULL;
995 }
996
997 if (timer_pending(&ctrl->activity_timer))
998 del_timer_sync(&ctrl->activity_timer);
999
1000 /* Clean up the DMA queue */
1001 spin_lock_bh(&ctrl->queue_lock);
1002 rsxx_cleanup_dma_queue(ctrl, &ctrl->queue, COMPLETE_DMA);
1003 spin_unlock_bh(&ctrl->queue_lock);
1004
1005 rsxx_dma_cancel(ctrl);
1006
1007 vfree(ctrl->trackers);
1008
1009 dma_free_coherent(&card->dev->dev, STATUS_BUFFER_SIZE8,
1010 ctrl->status.buf, ctrl->status.dma_addr);
1011 dma_free_coherent(&card->dev->dev, COMMAND_BUFFER_SIZE8,
1012 ctrl->cmd.buf, ctrl->cmd.dma_addr);
1013 }
1014 }
1015
rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo * card)1016 int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card)
1017 {
1018 int i;
1019 int j;
1020 int cnt;
1021 struct rsxx_dma *dma;
1022 struct list_head *issued_dmas;
1023
1024 issued_dmas = kcalloc(card->n_targets, sizeof(*issued_dmas),
1025 GFP_KERNEL);
1026 if (!issued_dmas)
1027 return -ENOMEM;
1028
1029 for (i = 0; i < card->n_targets; i++) {
1030 INIT_LIST_HEAD(&issued_dmas[i]);
1031 cnt = 0;
1032 for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
1033 dma = get_tracker_dma(card->ctrl[i].trackers, j);
1034 if (dma == NULL)
1035 continue;
1036
1037 if (dma->cmd == HW_CMD_BLK_WRITE)
1038 card->ctrl[i].stats.writes_issued--;
1039 else if (dma->cmd == HW_CMD_BLK_DISCARD)
1040 card->ctrl[i].stats.discards_issued--;
1041 else
1042 card->ctrl[i].stats.reads_issued--;
1043
1044 if (dma->cmd != HW_CMD_BLK_DISCARD) {
1045 dma_unmap_page(&card->dev->dev, dma->dma_addr,
1046 get_dma_size(dma),
1047 dma->cmd == HW_CMD_BLK_WRITE ?
1048 DMA_TO_DEVICE :
1049 DMA_FROM_DEVICE);
1050 }
1051
1052 list_add_tail(&dma->list, &issued_dmas[i]);
1053 push_tracker(card->ctrl[i].trackers, j);
1054 cnt++;
1055 }
1056
1057 spin_lock_bh(&card->ctrl[i].queue_lock);
1058 list_splice(&issued_dmas[i], &card->ctrl[i].queue);
1059
1060 atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
1061 card->ctrl[i].stats.sw_q_depth += cnt;
1062 card->ctrl[i].e_cnt = 0;
1063 spin_unlock_bh(&card->ctrl[i].queue_lock);
1064 }
1065
1066 kfree(issued_dmas);
1067
1068 return 0;
1069 }
1070
rsxx_dma_init(void)1071 int rsxx_dma_init(void)
1072 {
1073 rsxx_dma_pool = KMEM_CACHE(rsxx_dma, SLAB_HWCACHE_ALIGN);
1074 if (!rsxx_dma_pool)
1075 return -ENOMEM;
1076
1077 return 0;
1078 }
1079
1080
rsxx_dma_cleanup(void)1081 void rsxx_dma_cleanup(void)
1082 {
1083 kmem_cache_destroy(rsxx_dma_pool);
1084 }
1085
1086
