1 /*
2 * Filename: core.c
3 *
4 *
5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6 * Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7 *
8 * (C) Copyright 2013 IBM Corporation
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License as
12 * published by the Free Software Foundation; either version 2 of the
13 * License, or (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software Foundation,
22 * Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
24
25 #include <linux/kernel.h>
26 #include <linux/init.h>
27 #include <linux/interrupt.h>
28 #include <linux/module.h>
29 #include <linux/pci.h>
30 #include <linux/reboot.h>
31 #include <linux/slab.h>
32 #include <linux/bitops.h>
33 #include <linux/delay.h>
34 #include <linux/debugfs.h>
35 #include <linux/seq_file.h>
36
37 #include <linux/genhd.h>
38 #include <linux/idr.h>
39
40 #include "rsxx_priv.h"
41 #include "rsxx_cfg.h"
42
43 #define NO_LEGACY 0
44 #define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
45
46 MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
47 MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
48 MODULE_LICENSE("GPL");
49 MODULE_VERSION(DRIVER_VERSION);
50
51 static unsigned int force_legacy = NO_LEGACY;
52 module_param(force_legacy, uint, 0444);
53 MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
54
55 static unsigned int sync_start = 1;
56 module_param(sync_start, uint, 0444);
57 MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
58 "until the card startup has completed.");
59
60 static DEFINE_IDA(rsxx_disk_ida);
61
62 /* --------------------Debugfs Setup ------------------- */
63
rsxx_attr_pci_regs_show(struct seq_file * m,void * p)64 static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
65 {
66 struct rsxx_cardinfo *card = m->private;
67
68 seq_printf(m, "HWID 0x%08x\n",
69 ioread32(card->regmap + HWID));
70 seq_printf(m, "SCRATCH 0x%08x\n",
71 ioread32(card->regmap + SCRATCH));
72 seq_printf(m, "IER 0x%08x\n",
73 ioread32(card->regmap + IER));
74 seq_printf(m, "IPR 0x%08x\n",
75 ioread32(card->regmap + IPR));
76 seq_printf(m, "CREG_CMD 0x%08x\n",
77 ioread32(card->regmap + CREG_CMD));
78 seq_printf(m, "CREG_ADD 0x%08x\n",
79 ioread32(card->regmap + CREG_ADD));
80 seq_printf(m, "CREG_CNT 0x%08x\n",
81 ioread32(card->regmap + CREG_CNT));
82 seq_printf(m, "CREG_STAT 0x%08x\n",
83 ioread32(card->regmap + CREG_STAT));
84 seq_printf(m, "CREG_DATA0 0x%08x\n",
85 ioread32(card->regmap + CREG_DATA0));
86 seq_printf(m, "CREG_DATA1 0x%08x\n",
87 ioread32(card->regmap + CREG_DATA1));
88 seq_printf(m, "CREG_DATA2 0x%08x\n",
89 ioread32(card->regmap + CREG_DATA2));
90 seq_printf(m, "CREG_DATA3 0x%08x\n",
91 ioread32(card->regmap + CREG_DATA3));
92 seq_printf(m, "CREG_DATA4 0x%08x\n",
93 ioread32(card->regmap + CREG_DATA4));
94 seq_printf(m, "CREG_DATA5 0x%08x\n",
95 ioread32(card->regmap + CREG_DATA5));
96 seq_printf(m, "CREG_DATA6 0x%08x\n",
97 ioread32(card->regmap + CREG_DATA6));
98 seq_printf(m, "CREG_DATA7 0x%08x\n",
99 ioread32(card->regmap + CREG_DATA7));
100 seq_printf(m, "INTR_COAL 0x%08x\n",
101 ioread32(card->regmap + INTR_COAL));
102 seq_printf(m, "HW_ERROR 0x%08x\n",
103 ioread32(card->regmap + HW_ERROR));
104 seq_printf(m, "DEBUG0 0x%08x\n",
105 ioread32(card->regmap + PCI_DEBUG0));
106 seq_printf(m, "DEBUG1 0x%08x\n",
107 ioread32(card->regmap + PCI_DEBUG1));
108 seq_printf(m, "DEBUG2 0x%08x\n",
109 ioread32(card->regmap + PCI_DEBUG2));
110 seq_printf(m, "DEBUG3 0x%08x\n",
111 ioread32(card->regmap + PCI_DEBUG3));
112 seq_printf(m, "DEBUG4 0x%08x\n",
113 ioread32(card->regmap + PCI_DEBUG4));
114 seq_printf(m, "DEBUG5 0x%08x\n",
115 ioread32(card->regmap + PCI_DEBUG5));
116 seq_printf(m, "DEBUG6 0x%08x\n",
117 ioread32(card->regmap + PCI_DEBUG6));
118 seq_printf(m, "DEBUG7 0x%08x\n",
119 ioread32(card->regmap + PCI_DEBUG7));
120 seq_printf(m, "RECONFIG 0x%08x\n",
121 ioread32(card->regmap + PCI_RECONFIG));
122
123 return 0;
124 }
125
rsxx_attr_stats_show(struct seq_file * m,void * p)126 static int rsxx_attr_stats_show(struct seq_file *m, void *p)
127 {
128 struct rsxx_cardinfo *card = m->private;
129 int i;
130
131 for (i = 0; i < card->n_targets; i++) {
132 seq_printf(m, "Ctrl %d CRC Errors = %d\n",
133 i, card->ctrl[i].stats.crc_errors);
134 seq_printf(m, "Ctrl %d Hard Errors = %d\n",
135 i, card->ctrl[i].stats.hard_errors);
136 seq_printf(m, "Ctrl %d Soft Errors = %d\n",
137 i, card->ctrl[i].stats.soft_errors);
138 seq_printf(m, "Ctrl %d Writes Issued = %d\n",
139 i, card->ctrl[i].stats.writes_issued);
140 seq_printf(m, "Ctrl %d Writes Failed = %d\n",
141 i, card->ctrl[i].stats.writes_failed);
142 seq_printf(m, "Ctrl %d Reads Issued = %d\n",
143 i, card->ctrl[i].stats.reads_issued);
144 seq_printf(m, "Ctrl %d Reads Failed = %d\n",
145 i, card->ctrl[i].stats.reads_failed);
146 seq_printf(m, "Ctrl %d Reads Retried = %d\n",
147 i, card->ctrl[i].stats.reads_retried);
148 seq_printf(m, "Ctrl %d Discards Issued = %d\n",
149 i, card->ctrl[i].stats.discards_issued);
150 seq_printf(m, "Ctrl %d Discards Failed = %d\n",
151 i, card->ctrl[i].stats.discards_failed);
152 seq_printf(m, "Ctrl %d DMA SW Errors = %d\n",
153 i, card->ctrl[i].stats.dma_sw_err);
154 seq_printf(m, "Ctrl %d DMA HW Faults = %d\n",
155 i, card->ctrl[i].stats.dma_hw_fault);
156 seq_printf(m, "Ctrl %d DMAs Cancelled = %d\n",
157 i, card->ctrl[i].stats.dma_cancelled);
158 seq_printf(m, "Ctrl %d SW Queue Depth = %d\n",
159 i, card->ctrl[i].stats.sw_q_depth);
160 seq_printf(m, "Ctrl %d HW Queue Depth = %d\n",
161 i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
162 }
163
164 return 0;
165 }
166
rsxx_attr_stats_open(struct inode * inode,struct file * file)167 static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
168 {
169 return single_open(file, rsxx_attr_stats_show, inode->i_private);
170 }
171
rsxx_attr_pci_regs_open(struct inode * inode,struct file * file)172 static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
173 {
174 return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
175 }
176
rsxx_cram_read(struct file * fp,char __user * ubuf,size_t cnt,loff_t * ppos)177 static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
178 size_t cnt, loff_t *ppos)
179 {
180 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
181 char *buf;
182 ssize_t st;
183
184 buf = kzalloc(cnt, GFP_KERNEL);
185 if (!buf)
186 return -ENOMEM;
187
188 st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
189 if (!st)
190 st = copy_to_user(ubuf, buf, cnt);
191 kfree(buf);
192 if (st)
193 return st;
194 *ppos += cnt;
195 return cnt;
196 }
197
rsxx_cram_write(struct file * fp,const char __user * ubuf,size_t cnt,loff_t * ppos)198 static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
199 size_t cnt, loff_t *ppos)
200 {
201 struct rsxx_cardinfo *card = file_inode(fp)->i_private;
202 char *buf;
203 ssize_t st;
204
205 buf = memdup_user(ubuf, cnt);
206 if (IS_ERR(buf))
207 return PTR_ERR(buf);
208
209 st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
210 kfree(buf);
211 if (st)
212 return st;
213 *ppos += cnt;
214 return cnt;
215 }
216
217 static const struct file_operations debugfs_cram_fops = {
218 .owner = THIS_MODULE,
219 .read = rsxx_cram_read,
220 .write = rsxx_cram_write,
221 };
222
223 static const struct file_operations debugfs_stats_fops = {
224 .owner = THIS_MODULE,
225 .open = rsxx_attr_stats_open,
226 .read = seq_read,
227 .llseek = seq_lseek,
228 .release = single_release,
229 };
230
231 static const struct file_operations debugfs_pci_regs_fops = {
232 .owner = THIS_MODULE,
233 .open = rsxx_attr_pci_regs_open,
234 .read = seq_read,
235 .llseek = seq_lseek,
236 .release = single_release,
237 };
238
rsxx_debugfs_dev_new(struct rsxx_cardinfo * card)239 static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
240 {
241 struct dentry *debugfs_stats;
242 struct dentry *debugfs_pci_regs;
243 struct dentry *debugfs_cram;
244
245 card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
246 if (IS_ERR_OR_NULL(card->debugfs_dir))
247 goto failed_debugfs_dir;
248
249 debugfs_stats = debugfs_create_file("stats", 0444,
250 card->debugfs_dir, card,
251 &debugfs_stats_fops);
252 if (IS_ERR_OR_NULL(debugfs_stats))
253 goto failed_debugfs_stats;
254
255 debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
256 card->debugfs_dir, card,
257 &debugfs_pci_regs_fops);
258 if (IS_ERR_OR_NULL(debugfs_pci_regs))
259 goto failed_debugfs_pci_regs;
260
261 debugfs_cram = debugfs_create_file("cram", 0644,
262 card->debugfs_dir, card,
263 &debugfs_cram_fops);
264 if (IS_ERR_OR_NULL(debugfs_cram))
265 goto failed_debugfs_cram;
266
267 return;
268 failed_debugfs_cram:
269 debugfs_remove(debugfs_pci_regs);
270 failed_debugfs_pci_regs:
271 debugfs_remove(debugfs_stats);
272 failed_debugfs_stats:
273 debugfs_remove(card->debugfs_dir);
274 failed_debugfs_dir:
275 card->debugfs_dir = NULL;
276 }
277
278 /*----------------- Interrupt Control & Handling -------------------*/
279
rsxx_mask_interrupts(struct rsxx_cardinfo * card)280 static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
281 {
282 card->isr_mask = 0;
283 card->ier_mask = 0;
284 }
285
__enable_intr(unsigned int * mask,unsigned int intr)286 static void __enable_intr(unsigned int *mask, unsigned int intr)
287 {
288 *mask |= intr;
289 }
290
__disable_intr(unsigned int * mask,unsigned int intr)291 static void __disable_intr(unsigned int *mask, unsigned int intr)
292 {
293 *mask &= ~intr;
294 }
295
296 /*
297 * NOTE: Disabling the IER will disable the hardware interrupt.
298 * Disabling the ISR will disable the software handling of the ISR bit.
299 *
300 * Enable/Disable interrupt functions assume the card->irq_lock
301 * is held by the caller.
302 */
rsxx_enable_ier(struct rsxx_cardinfo * card,unsigned int intr)303 void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
304 {
305 if (unlikely(card->halt) ||
306 unlikely(card->eeh_state))
307 return;
308
309 __enable_intr(&card->ier_mask, intr);
310 iowrite32(card->ier_mask, card->regmap + IER);
311 }
312
rsxx_disable_ier(struct rsxx_cardinfo * card,unsigned int intr)313 void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
314 {
315 if (unlikely(card->eeh_state))
316 return;
317
318 __disable_intr(&card->ier_mask, intr);
319 iowrite32(card->ier_mask, card->regmap + IER);
320 }
321
rsxx_enable_ier_and_isr(struct rsxx_cardinfo * card,unsigned int intr)322 void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
323 unsigned int intr)
324 {
325 if (unlikely(card->halt) ||
326 unlikely(card->eeh_state))
327 return;
328
329 __enable_intr(&card->isr_mask, intr);
330 __enable_intr(&card->ier_mask, intr);
331 iowrite32(card->ier_mask, card->regmap + IER);
332 }
rsxx_disable_ier_and_isr(struct rsxx_cardinfo * card,unsigned int intr)333 void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
334 unsigned int intr)
335 {
336 if (unlikely(card->eeh_state))
337 return;
338
339 __disable_intr(&card->isr_mask, intr);
340 __disable_intr(&card->ier_mask, intr);
341 iowrite32(card->ier_mask, card->regmap + IER);
342 }
343
rsxx_isr(int irq,void * pdata)344 static irqreturn_t rsxx_isr(int irq, void *pdata)
345 {
346 struct rsxx_cardinfo *card = pdata;
347 unsigned int isr;
348 int handled = 0;
349 int reread_isr;
350 int i;
351
352 spin_lock(&card->irq_lock);
353
354 do {
355 reread_isr = 0;
356
357 if (unlikely(card->eeh_state))
358 break;
359
360 isr = ioread32(card->regmap + ISR);
361 if (isr == 0xffffffff) {
362 /*
363 * A few systems seem to have an intermittent issue
364 * where PCI reads return all Fs, but retrying the read
365 * a little later will return as expected.
366 */
367 dev_info(CARD_TO_DEV(card),
368 "ISR = 0xFFFFFFFF, retrying later\n");
369 break;
370 }
371
372 isr &= card->isr_mask;
373 if (!isr)
374 break;
375
376 for (i = 0; i < card->n_targets; i++) {
377 if (isr & CR_INTR_DMA(i)) {
378 if (card->ier_mask & CR_INTR_DMA(i)) {
379 rsxx_disable_ier(card, CR_INTR_DMA(i));
380 reread_isr = 1;
381 }
382 queue_work(card->ctrl[i].done_wq,
383 &card->ctrl[i].dma_done_work);
384 handled++;
385 }
386 }
387
388 if (isr & CR_INTR_CREG) {
389 queue_work(card->creg_ctrl.creg_wq,
390 &card->creg_ctrl.done_work);
391 handled++;
392 }
393
394 if (isr & CR_INTR_EVENT) {
395 queue_work(card->event_wq, &card->event_work);
396 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
397 handled++;
398 }
399 } while (reread_isr);
400
401 spin_unlock(&card->irq_lock);
402
403 return handled ? IRQ_HANDLED : IRQ_NONE;
404 }
405
406 /*----------------- Card Event Handler -------------------*/
rsxx_card_state_to_str(unsigned int state)407 static const char * const rsxx_card_state_to_str(unsigned int state)
408 {
409 static const char * const state_strings[] = {
410 "Unknown", "Shutdown", "Starting", "Formatting",
411 "Uninitialized", "Good", "Shutting Down",
412 "Fault", "Read Only Fault", "dStroying"
413 };
414
415 return state_strings[ffs(state)];
416 }
417
card_state_change(struct rsxx_cardinfo * card,unsigned int new_state)418 static void card_state_change(struct rsxx_cardinfo *card,
419 unsigned int new_state)
420 {
421 int st;
422
423 dev_info(CARD_TO_DEV(card),
424 "card state change detected.(%s -> %s)\n",
425 rsxx_card_state_to_str(card->state),
426 rsxx_card_state_to_str(new_state));
427
428 card->state = new_state;
429
430 /* Don't attach DMA interfaces if the card has an invalid config */
431 if (!card->config_valid)
432 return;
433
434 switch (new_state) {
435 case CARD_STATE_RD_ONLY_FAULT:
436 dev_crit(CARD_TO_DEV(card),
437 "Hardware has entered read-only mode!\n");
438 /*
439 * Fall through so the DMA devices can be attached and
440 * the user can attempt to pull off their data.
441 */
442 case CARD_STATE_GOOD:
443 st = rsxx_get_card_size8(card, &card->size8);
444 if (st)
445 dev_err(CARD_TO_DEV(card),
446 "Failed attaching DMA devices\n");
447
448 if (card->config_valid)
449 set_capacity(card->gendisk, card->size8 >> 9);
450 break;
451
452 case CARD_STATE_FAULT:
453 dev_crit(CARD_TO_DEV(card),
454 "Hardware Fault reported!\n");
455 /* Fall through. */
456
457 /* Everything else, detach DMA interface if it's attached. */
458 case CARD_STATE_SHUTDOWN:
459 case CARD_STATE_STARTING:
460 case CARD_STATE_FORMATTING:
461 case CARD_STATE_UNINITIALIZED:
462 case CARD_STATE_SHUTTING_DOWN:
463 /*
464 * dStroy is a term coined by marketing to represent the low level
465 * secure erase.
466 */
467 case CARD_STATE_DSTROYING:
468 set_capacity(card->gendisk, 0);
469 break;
470 }
471 }
472
card_event_handler(struct work_struct * work)473 static void card_event_handler(struct work_struct *work)
474 {
475 struct rsxx_cardinfo *card;
476 unsigned int state;
477 unsigned long flags;
478 int st;
479
480 card = container_of(work, struct rsxx_cardinfo, event_work);
481
482 if (unlikely(card->halt))
483 return;
484
485 /*
486 * Enable the interrupt now to avoid any weird race conditions where a
487 * state change might occur while rsxx_get_card_state() is
488 * processing a returned creg cmd.
489 */
490 spin_lock_irqsave(&card->irq_lock, flags);
491 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
492 spin_unlock_irqrestore(&card->irq_lock, flags);
493
494 st = rsxx_get_card_state(card, &state);
495 if (st) {
496 dev_info(CARD_TO_DEV(card),
497 "Failed reading state after event.\n");
498 return;
499 }
500
501 if (card->state != state)
502 card_state_change(card, state);
503
504 if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
505 rsxx_read_hw_log(card);
506 }
507
508 /*----------------- Card Operations -------------------*/
card_shutdown(struct rsxx_cardinfo * card)509 static int card_shutdown(struct rsxx_cardinfo *card)
510 {
511 unsigned int state;
512 signed long start;
513 const int timeout = msecs_to_jiffies(120000);
514 int st;
515
516 /* We can't issue a shutdown if the card is in a transition state */
517 start = jiffies;
518 do {
519 st = rsxx_get_card_state(card, &state);
520 if (st)
521 return st;
522 } while (state == CARD_STATE_STARTING &&
523 (jiffies - start < timeout));
524
525 if (state == CARD_STATE_STARTING)
526 return -ETIMEDOUT;
527
528 /* Only issue a shutdown if we need to */
529 if ((state != CARD_STATE_SHUTTING_DOWN) &&
530 (state != CARD_STATE_SHUTDOWN)) {
531 st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
532 if (st)
533 return st;
534 }
535
536 start = jiffies;
537 do {
538 st = rsxx_get_card_state(card, &state);
539 if (st)
540 return st;
541 } while (state != CARD_STATE_SHUTDOWN &&
542 (jiffies - start < timeout));
543
544 if (state != CARD_STATE_SHUTDOWN)
545 return -ETIMEDOUT;
546
547 return 0;
548 }
549
rsxx_eeh_frozen(struct pci_dev * dev)550 static int rsxx_eeh_frozen(struct pci_dev *dev)
551 {
552 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
553 int i;
554 int st;
555
556 dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
557
558 card->eeh_state = 1;
559 rsxx_mask_interrupts(card);
560
561 /*
562 * We need to guarantee that the write for eeh_state and masking
563 * interrupts does not become reordered. This will prevent a possible
564 * race condition with the EEH code.
565 */
566 wmb();
567
568 pci_disable_device(dev);
569
570 st = rsxx_eeh_save_issued_dmas(card);
571 if (st)
572 return st;
573
574 rsxx_eeh_save_issued_creg(card);
575
576 for (i = 0; i < card->n_targets; i++) {
577 if (card->ctrl[i].status.buf)
578 pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
579 card->ctrl[i].status.buf,
580 card->ctrl[i].status.dma_addr);
581 if (card->ctrl[i].cmd.buf)
582 pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
583 card->ctrl[i].cmd.buf,
584 card->ctrl[i].cmd.dma_addr);
585 }
586
587 return 0;
588 }
589
rsxx_eeh_failure(struct pci_dev * dev)590 static void rsxx_eeh_failure(struct pci_dev *dev)
591 {
592 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
593 int i;
594 int cnt = 0;
595
596 dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
597
598 card->eeh_state = 1;
599 card->halt = 1;
600
601 for (i = 0; i < card->n_targets; i++) {
602 spin_lock_bh(&card->ctrl[i].queue_lock);
603 cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
604 &card->ctrl[i].queue,
605 COMPLETE_DMA);
606 spin_unlock_bh(&card->ctrl[i].queue_lock);
607
608 cnt += rsxx_dma_cancel(&card->ctrl[i]);
609
610 if (cnt)
611 dev_info(CARD_TO_DEV(card),
612 "Freed %d queued DMAs on channel %d\n",
613 cnt, card->ctrl[i].id);
614 }
615 }
616
rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo * card)617 static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
618 {
619 unsigned int status;
620 int iter = 0;
621
622 /* We need to wait for the hardware to reset */
623 while (iter++ < 10) {
624 status = ioread32(card->regmap + PCI_RECONFIG);
625
626 if (status & RSXX_FLUSH_BUSY) {
627 ssleep(1);
628 continue;
629 }
630
631 if (status & RSXX_FLUSH_TIMEOUT)
632 dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
633 return 0;
634 }
635
636 /* Hardware failed resetting itself. */
637 return -1;
638 }
639
rsxx_error_detected(struct pci_dev * dev,enum pci_channel_state error)640 static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
641 enum pci_channel_state error)
642 {
643 int st;
644
645 if (dev->revision < RSXX_EEH_SUPPORT)
646 return PCI_ERS_RESULT_NONE;
647
648 if (error == pci_channel_io_perm_failure) {
649 rsxx_eeh_failure(dev);
650 return PCI_ERS_RESULT_DISCONNECT;
651 }
652
653 st = rsxx_eeh_frozen(dev);
654 if (st) {
655 dev_err(&dev->dev, "Slot reset setup failed\n");
656 rsxx_eeh_failure(dev);
657 return PCI_ERS_RESULT_DISCONNECT;
658 }
659
660 return PCI_ERS_RESULT_NEED_RESET;
661 }
662
rsxx_slot_reset(struct pci_dev * dev)663 static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
664 {
665 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
666 unsigned long flags;
667 int i;
668 int st;
669
670 dev_warn(&dev->dev,
671 "IBM Flash Adapter PCI: recovering from slot reset.\n");
672
673 st = pci_enable_device(dev);
674 if (st)
675 goto failed_hw_setup;
676
677 pci_set_master(dev);
678
679 st = rsxx_eeh_fifo_flush_poll(card);
680 if (st)
681 goto failed_hw_setup;
682
683 rsxx_dma_queue_reset(card);
684
685 for (i = 0; i < card->n_targets; i++) {
686 st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
687 if (st)
688 goto failed_hw_buffers_init;
689 }
690
691 if (card->config_valid)
692 rsxx_dma_configure(card);
693
694 /* Clears the ISR register from spurious interrupts */
695 st = ioread32(card->regmap + ISR);
696
697 card->eeh_state = 0;
698
699 spin_lock_irqsave(&card->irq_lock, flags);
700 if (card->n_targets & RSXX_MAX_TARGETS)
701 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
702 else
703 rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
704 spin_unlock_irqrestore(&card->irq_lock, flags);
705
706 rsxx_kick_creg_queue(card);
707
708 for (i = 0; i < card->n_targets; i++) {
709 spin_lock(&card->ctrl[i].queue_lock);
710 if (list_empty(&card->ctrl[i].queue)) {
711 spin_unlock(&card->ctrl[i].queue_lock);
712 continue;
713 }
714 spin_unlock(&card->ctrl[i].queue_lock);
715
716 queue_work(card->ctrl[i].issue_wq,
717 &card->ctrl[i].issue_dma_work);
718 }
719
720 dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
721
722 return PCI_ERS_RESULT_RECOVERED;
723
724 failed_hw_buffers_init:
725 for (i = 0; i < card->n_targets; i++) {
726 if (card->ctrl[i].status.buf)
727 pci_free_consistent(card->dev,
728 STATUS_BUFFER_SIZE8,
729 card->ctrl[i].status.buf,
730 card->ctrl[i].status.dma_addr);
731 if (card->ctrl[i].cmd.buf)
732 pci_free_consistent(card->dev,
733 COMMAND_BUFFER_SIZE8,
734 card->ctrl[i].cmd.buf,
735 card->ctrl[i].cmd.dma_addr);
736 }
737 failed_hw_setup:
738 rsxx_eeh_failure(dev);
739 return PCI_ERS_RESULT_DISCONNECT;
740
741 }
742
743 /*----------------- Driver Initialization & Setup -------------------*/
744 /* Returns: 0 if the driver is compatible with the device
745 -1 if the driver is NOT compatible with the device */
rsxx_compatibility_check(struct rsxx_cardinfo * card)746 static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
747 {
748 unsigned char pci_rev;
749
750 pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
751
752 if (pci_rev > RS70_PCI_REV_SUPPORTED)
753 return -1;
754 return 0;
755 }
756
rsxx_pci_probe(struct pci_dev * dev,const struct pci_device_id * id)757 static int rsxx_pci_probe(struct pci_dev *dev,
758 const struct pci_device_id *id)
759 {
760 struct rsxx_cardinfo *card;
761 int st;
762 unsigned int sync_timeout;
763
764 dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
765
766 card = kzalloc(sizeof(*card), GFP_KERNEL);
767 if (!card)
768 return -ENOMEM;
769
770 card->dev = dev;
771 pci_set_drvdata(dev, card);
772
773 st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
774 if (st < 0)
775 goto failed_ida_get;
776 card->disk_id = st;
777
778 st = pci_enable_device(dev);
779 if (st)
780 goto failed_enable;
781
782 pci_set_master(dev);
783 pci_set_dma_max_seg_size(dev, RSXX_HW_BLK_SIZE);
784
785 st = pci_set_dma_mask(dev, DMA_BIT_MASK(64));
786 if (st) {
787 dev_err(CARD_TO_DEV(card),
788 "No usable DMA configuration,aborting\n");
789 goto failed_dma_mask;
790 }
791
792 st = pci_request_regions(dev, DRIVER_NAME);
793 if (st) {
794 dev_err(CARD_TO_DEV(card),
795 "Failed to request memory region\n");
796 goto failed_request_regions;
797 }
798
799 if (pci_resource_len(dev, 0) == 0) {
800 dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
801 st = -ENOMEM;
802 goto failed_iomap;
803 }
804
805 card->regmap = pci_iomap(dev, 0, 0);
806 if (!card->regmap) {
807 dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
808 st = -ENOMEM;
809 goto failed_iomap;
810 }
811
812 spin_lock_init(&card->irq_lock);
813 card->halt = 0;
814 card->eeh_state = 0;
815
816 spin_lock_irq(&card->irq_lock);
817 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
818 spin_unlock_irq(&card->irq_lock);
819
820 if (!force_legacy) {
821 st = pci_enable_msi(dev);
822 if (st)
823 dev_warn(CARD_TO_DEV(card),
824 "Failed to enable MSI\n");
825 }
826
827 st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
828 DRIVER_NAME, card);
829 if (st) {
830 dev_err(CARD_TO_DEV(card),
831 "Failed requesting IRQ%d\n", dev->irq);
832 goto failed_irq;
833 }
834
835 /************* Setup Processor Command Interface *************/
836 st = rsxx_creg_setup(card);
837 if (st) {
838 dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
839 goto failed_creg_setup;
840 }
841
842 spin_lock_irq(&card->irq_lock);
843 rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
844 spin_unlock_irq(&card->irq_lock);
845
846 st = rsxx_compatibility_check(card);
847 if (st) {
848 dev_warn(CARD_TO_DEV(card),
849 "Incompatible driver detected. Please update the driver.\n");
850 st = -EINVAL;
851 goto failed_compatiblity_check;
852 }
853
854 /************* Load Card Config *************/
855 st = rsxx_load_config(card);
856 if (st)
857 dev_err(CARD_TO_DEV(card),
858 "Failed loading card config\n");
859
860 /************* Setup DMA Engine *************/
861 st = rsxx_get_num_targets(card, &card->n_targets);
862 if (st)
863 dev_info(CARD_TO_DEV(card),
864 "Failed reading the number of DMA targets\n");
865
866 card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
867 GFP_KERNEL);
868 if (!card->ctrl) {
869 st = -ENOMEM;
870 goto failed_dma_setup;
871 }
872
873 st = rsxx_dma_setup(card);
874 if (st) {
875 dev_info(CARD_TO_DEV(card),
876 "Failed to setup DMA engine\n");
877 goto failed_dma_setup;
878 }
879
880 /************* Setup Card Event Handler *************/
881 card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
882 if (!card->event_wq) {
883 dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
884 goto failed_event_handler;
885 }
886
887 INIT_WORK(&card->event_work, card_event_handler);
888
889 st = rsxx_setup_dev(card);
890 if (st)
891 goto failed_create_dev;
892
893 rsxx_get_card_state(card, &card->state);
894
895 dev_info(CARD_TO_DEV(card),
896 "card state: %s\n",
897 rsxx_card_state_to_str(card->state));
898
899 /*
900 * Now that the DMA Engine and devices have been setup,
901 * we can enable the event interrupt(it kicks off actions in
902 * those layers so we couldn't enable it right away.)
903 */
904 spin_lock_irq(&card->irq_lock);
905 rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
906 spin_unlock_irq(&card->irq_lock);
907
908 if (card->state == CARD_STATE_SHUTDOWN) {
909 st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
910 if (st)
911 dev_crit(CARD_TO_DEV(card),
912 "Failed issuing card startup\n");
913 if (sync_start) {
914 sync_timeout = SYNC_START_TIMEOUT;
915
916 dev_info(CARD_TO_DEV(card),
917 "Waiting for card to startup\n");
918
919 do {
920 ssleep(1);
921 sync_timeout--;
922
923 rsxx_get_card_state(card, &card->state);
924 } while (sync_timeout &&
925 (card->state == CARD_STATE_STARTING));
926
927 if (card->state == CARD_STATE_STARTING) {
928 dev_warn(CARD_TO_DEV(card),
929 "Card startup timed out\n");
930 card->size8 = 0;
931 } else {
932 dev_info(CARD_TO_DEV(card),
933 "card state: %s\n",
934 rsxx_card_state_to_str(card->state));
935 st = rsxx_get_card_size8(card, &card->size8);
936 if (st)
937 card->size8 = 0;
938 }
939 }
940 } else if (card->state == CARD_STATE_GOOD ||
941 card->state == CARD_STATE_RD_ONLY_FAULT) {
942 st = rsxx_get_card_size8(card, &card->size8);
943 if (st)
944 card->size8 = 0;
945 }
946
947 rsxx_attach_dev(card);
948
949 /************* Setup Debugfs *************/
950 rsxx_debugfs_dev_new(card);
951
952 return 0;
953
954 failed_create_dev:
955 destroy_workqueue(card->event_wq);
956 card->event_wq = NULL;
957 failed_event_handler:
958 rsxx_dma_destroy(card);
959 failed_dma_setup:
960 failed_compatiblity_check:
961 destroy_workqueue(card->creg_ctrl.creg_wq);
962 card->creg_ctrl.creg_wq = NULL;
963 failed_creg_setup:
964 spin_lock_irq(&card->irq_lock);
965 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
966 spin_unlock_irq(&card->irq_lock);
967 free_irq(dev->irq, card);
968 if (!force_legacy)
969 pci_disable_msi(dev);
970 failed_irq:
971 pci_iounmap(dev, card->regmap);
972 failed_iomap:
973 pci_release_regions(dev);
974 failed_request_regions:
975 failed_dma_mask:
976 pci_disable_device(dev);
977 failed_enable:
978 ida_free(&rsxx_disk_ida, card->disk_id);
979 failed_ida_get:
980 kfree(card);
981
982 return st;
983 }
984
rsxx_pci_remove(struct pci_dev * dev)985 static void rsxx_pci_remove(struct pci_dev *dev)
986 {
987 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
988 unsigned long flags;
989 int st;
990 int i;
991
992 if (!card)
993 return;
994
995 dev_info(CARD_TO_DEV(card),
996 "Removing PCI-Flash SSD.\n");
997
998 rsxx_detach_dev(card);
999
1000 for (i = 0; i < card->n_targets; i++) {
1001 spin_lock_irqsave(&card->irq_lock, flags);
1002 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1003 spin_unlock_irqrestore(&card->irq_lock, flags);
1004 }
1005
1006 st = card_shutdown(card);
1007 if (st)
1008 dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
1009
1010 /* Sync outstanding event handlers. */
1011 spin_lock_irqsave(&card->irq_lock, flags);
1012 rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
1013 spin_unlock_irqrestore(&card->irq_lock, flags);
1014
1015 cancel_work_sync(&card->event_work);
1016
1017 rsxx_destroy_dev(card);
1018 rsxx_dma_destroy(card);
1019
1020 spin_lock_irqsave(&card->irq_lock, flags);
1021 rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1022 spin_unlock_irqrestore(&card->irq_lock, flags);
1023
1024 /* Prevent work_structs from re-queuing themselves. */
1025 card->halt = 1;
1026
1027 debugfs_remove_recursive(card->debugfs_dir);
1028
1029 free_irq(dev->irq, card);
1030
1031 if (!force_legacy)
1032 pci_disable_msi(dev);
1033
1034 rsxx_creg_destroy(card);
1035
1036 pci_iounmap(dev, card->regmap);
1037
1038 pci_disable_device(dev);
1039 pci_release_regions(dev);
1040
1041 ida_free(&rsxx_disk_ida, card->disk_id);
1042 kfree(card);
1043 }
1044
rsxx_pci_suspend(struct pci_dev * dev,pm_message_t state)1045 static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1046 {
1047 /* We don't support suspend at this time. */
1048 return -ENOSYS;
1049 }
1050
rsxx_pci_shutdown(struct pci_dev * dev)1051 static void rsxx_pci_shutdown(struct pci_dev *dev)
1052 {
1053 struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1054 unsigned long flags;
1055 int i;
1056
1057 if (!card)
1058 return;
1059
1060 dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1061
1062 rsxx_detach_dev(card);
1063
1064 for (i = 0; i < card->n_targets; i++) {
1065 spin_lock_irqsave(&card->irq_lock, flags);
1066 rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1067 spin_unlock_irqrestore(&card->irq_lock, flags);
1068 }
1069
1070 card_shutdown(card);
1071 }
1072
1073 static const struct pci_error_handlers rsxx_err_handler = {
1074 .error_detected = rsxx_error_detected,
1075 .slot_reset = rsxx_slot_reset,
1076 };
1077
1078 static const struct pci_device_id rsxx_pci_ids[] = {
1079 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1080 {PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
1081 {0,},
1082 };
1083
1084 MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1085
1086 static struct pci_driver rsxx_pci_driver = {
1087 .name = DRIVER_NAME,
1088 .id_table = rsxx_pci_ids,
1089 .probe = rsxx_pci_probe,
1090 .remove = rsxx_pci_remove,
1091 .suspend = rsxx_pci_suspend,
1092 .shutdown = rsxx_pci_shutdown,
1093 .err_handler = &rsxx_err_handler,
1094 };
1095
rsxx_core_init(void)1096 static int __init rsxx_core_init(void)
1097 {
1098 int st;
1099
1100 st = rsxx_dev_init();
1101 if (st)
1102 return st;
1103
1104 st = rsxx_dma_init();
1105 if (st)
1106 goto dma_init_failed;
1107
1108 st = rsxx_creg_init();
1109 if (st)
1110 goto creg_init_failed;
1111
1112 return pci_register_driver(&rsxx_pci_driver);
1113
1114 creg_init_failed:
1115 rsxx_dma_cleanup();
1116 dma_init_failed:
1117 rsxx_dev_cleanup();
1118
1119 return st;
1120 }
1121
rsxx_core_cleanup(void)1122 static void __exit rsxx_core_cleanup(void)
1123 {
1124 pci_unregister_driver(&rsxx_pci_driver);
1125 rsxx_creg_cleanup();
1126 rsxx_dma_cleanup();
1127 rsxx_dev_cleanup();
1128 }
1129
1130 module_init(rsxx_core_init);
1131 module_exit(rsxx_core_cleanup);
1132
