1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Filename: core.c
4 *
5 * Authors: Joshua Morris <josh.h.morris@us.ibm.com>
6 *	Philip Kelleher <pjk1939@linux.vnet.ibm.com>
7 *
8 * (C) Copyright 2013 IBM Corporation
9 */
10 
11 #include <linux/kernel.h>
12 #include <linux/init.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/pci.h>
16 #include <linux/reboot.h>
17 #include <linux/slab.h>
18 #include <linux/bitops.h>
19 #include <linux/delay.h>
20 #include <linux/debugfs.h>
21 #include <linux/seq_file.h>
22 
23 #include <linux/genhd.h>
24 #include <linux/idr.h>
25 
26 #include "rsxx_priv.h"
27 #include "rsxx_cfg.h"
28 
29 #define NO_LEGACY 0
30 #define SYNC_START_TIMEOUT (10 * 60) /* 10 minutes */
31 
32 MODULE_DESCRIPTION("IBM Flash Adapter 900GB Full Height Device Driver");
33 MODULE_AUTHOR("Joshua Morris/Philip Kelleher, IBM");
34 MODULE_LICENSE("GPL");
35 MODULE_VERSION(DRIVER_VERSION);
36 
37 static unsigned int force_legacy = NO_LEGACY;
38 module_param(force_legacy, uint, 0444);
39 MODULE_PARM_DESC(force_legacy, "Force the use of legacy type PCI interrupts");
40 
41 static unsigned int sync_start = 1;
42 module_param(sync_start, uint, 0444);
43 MODULE_PARM_DESC(sync_start, "On by Default: Driver load will not complete "
44 			     "until the card startup has completed.");
45 
46 static DEFINE_IDA(rsxx_disk_ida);
47 
48 /* --------------------Debugfs Setup ------------------- */
49 
rsxx_attr_pci_regs_show(struct seq_file * m,void * p)50 static int rsxx_attr_pci_regs_show(struct seq_file *m, void *p)
51 {
52 	struct rsxx_cardinfo *card = m->private;
53 
54 	seq_printf(m, "HWID		0x%08x\n",
55 					ioread32(card->regmap + HWID));
56 	seq_printf(m, "SCRATCH		0x%08x\n",
57 					ioread32(card->regmap + SCRATCH));
58 	seq_printf(m, "IER		0x%08x\n",
59 					ioread32(card->regmap + IER));
60 	seq_printf(m, "IPR		0x%08x\n",
61 					ioread32(card->regmap + IPR));
62 	seq_printf(m, "CREG_CMD		0x%08x\n",
63 					ioread32(card->regmap + CREG_CMD));
64 	seq_printf(m, "CREG_ADD		0x%08x\n",
65 					ioread32(card->regmap + CREG_ADD));
66 	seq_printf(m, "CREG_CNT		0x%08x\n",
67 					ioread32(card->regmap + CREG_CNT));
68 	seq_printf(m, "CREG_STAT	0x%08x\n",
69 					ioread32(card->regmap + CREG_STAT));
70 	seq_printf(m, "CREG_DATA0	0x%08x\n",
71 					ioread32(card->regmap + CREG_DATA0));
72 	seq_printf(m, "CREG_DATA1	0x%08x\n",
73 					ioread32(card->regmap + CREG_DATA1));
74 	seq_printf(m, "CREG_DATA2	0x%08x\n",
75 					ioread32(card->regmap + CREG_DATA2));
76 	seq_printf(m, "CREG_DATA3	0x%08x\n",
77 					ioread32(card->regmap + CREG_DATA3));
78 	seq_printf(m, "CREG_DATA4	0x%08x\n",
79 					ioread32(card->regmap + CREG_DATA4));
80 	seq_printf(m, "CREG_DATA5	0x%08x\n",
81 					ioread32(card->regmap + CREG_DATA5));
82 	seq_printf(m, "CREG_DATA6	0x%08x\n",
83 					ioread32(card->regmap + CREG_DATA6));
84 	seq_printf(m, "CREG_DATA7	0x%08x\n",
85 					ioread32(card->regmap + CREG_DATA7));
86 	seq_printf(m, "INTR_COAL	0x%08x\n",
87 					ioread32(card->regmap + INTR_COAL));
88 	seq_printf(m, "HW_ERROR		0x%08x\n",
89 					ioread32(card->regmap + HW_ERROR));
90 	seq_printf(m, "DEBUG0		0x%08x\n",
91 					ioread32(card->regmap + PCI_DEBUG0));
92 	seq_printf(m, "DEBUG1		0x%08x\n",
93 					ioread32(card->regmap + PCI_DEBUG1));
94 	seq_printf(m, "DEBUG2		0x%08x\n",
95 					ioread32(card->regmap + PCI_DEBUG2));
96 	seq_printf(m, "DEBUG3		0x%08x\n",
97 					ioread32(card->regmap + PCI_DEBUG3));
98 	seq_printf(m, "DEBUG4		0x%08x\n",
99 					ioread32(card->regmap + PCI_DEBUG4));
100 	seq_printf(m, "DEBUG5		0x%08x\n",
101 					ioread32(card->regmap + PCI_DEBUG5));
102 	seq_printf(m, "DEBUG6		0x%08x\n",
103 					ioread32(card->regmap + PCI_DEBUG6));
104 	seq_printf(m, "DEBUG7		0x%08x\n",
105 					ioread32(card->regmap + PCI_DEBUG7));
106 	seq_printf(m, "RECONFIG		0x%08x\n",
107 					ioread32(card->regmap + PCI_RECONFIG));
108 
109 	return 0;
110 }
111 
rsxx_attr_stats_show(struct seq_file * m,void * p)112 static int rsxx_attr_stats_show(struct seq_file *m, void *p)
113 {
114 	struct rsxx_cardinfo *card = m->private;
115 	int i;
116 
117 	for (i = 0; i < card->n_targets; i++) {
118 		seq_printf(m, "Ctrl %d CRC Errors	= %d\n",
119 				i, card->ctrl[i].stats.crc_errors);
120 		seq_printf(m, "Ctrl %d Hard Errors	= %d\n",
121 				i, card->ctrl[i].stats.hard_errors);
122 		seq_printf(m, "Ctrl %d Soft Errors	= %d\n",
123 				i, card->ctrl[i].stats.soft_errors);
124 		seq_printf(m, "Ctrl %d Writes Issued	= %d\n",
125 				i, card->ctrl[i].stats.writes_issued);
126 		seq_printf(m, "Ctrl %d Writes Failed	= %d\n",
127 				i, card->ctrl[i].stats.writes_failed);
128 		seq_printf(m, "Ctrl %d Reads Issued	= %d\n",
129 				i, card->ctrl[i].stats.reads_issued);
130 		seq_printf(m, "Ctrl %d Reads Failed	= %d\n",
131 				i, card->ctrl[i].stats.reads_failed);
132 		seq_printf(m, "Ctrl %d Reads Retried	= %d\n",
133 				i, card->ctrl[i].stats.reads_retried);
134 		seq_printf(m, "Ctrl %d Discards Issued	= %d\n",
135 				i, card->ctrl[i].stats.discards_issued);
136 		seq_printf(m, "Ctrl %d Discards Failed	= %d\n",
137 				i, card->ctrl[i].stats.discards_failed);
138 		seq_printf(m, "Ctrl %d DMA SW Errors	= %d\n",
139 				i, card->ctrl[i].stats.dma_sw_err);
140 		seq_printf(m, "Ctrl %d DMA HW Faults	= %d\n",
141 				i, card->ctrl[i].stats.dma_hw_fault);
142 		seq_printf(m, "Ctrl %d DMAs Cancelled	= %d\n",
143 				i, card->ctrl[i].stats.dma_cancelled);
144 		seq_printf(m, "Ctrl %d SW Queue Depth	= %d\n",
145 				i, card->ctrl[i].stats.sw_q_depth);
146 		seq_printf(m, "Ctrl %d HW Queue Depth	= %d\n",
147 			i, atomic_read(&card->ctrl[i].stats.hw_q_depth));
148 	}
149 
150 	return 0;
151 }
152 
rsxx_attr_stats_open(struct inode * inode,struct file * file)153 static int rsxx_attr_stats_open(struct inode *inode, struct file *file)
154 {
155 	return single_open(file, rsxx_attr_stats_show, inode->i_private);
156 }
157 
rsxx_attr_pci_regs_open(struct inode * inode,struct file * file)158 static int rsxx_attr_pci_regs_open(struct inode *inode, struct file *file)
159 {
160 	return single_open(file, rsxx_attr_pci_regs_show, inode->i_private);
161 }
162 
rsxx_cram_read(struct file * fp,char __user * ubuf,size_t cnt,loff_t * ppos)163 static ssize_t rsxx_cram_read(struct file *fp, char __user *ubuf,
164 			      size_t cnt, loff_t *ppos)
165 {
166 	struct rsxx_cardinfo *card = file_inode(fp)->i_private;
167 	char *buf;
168 	ssize_t st;
169 
170 	buf = kzalloc(cnt, GFP_KERNEL);
171 	if (!buf)
172 		return -ENOMEM;
173 
174 	st = rsxx_creg_read(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
175 	if (!st)
176 		st = copy_to_user(ubuf, buf, cnt);
177 	kfree(buf);
178 	if (st)
179 		return st;
180 	*ppos += cnt;
181 	return cnt;
182 }
183 
rsxx_cram_write(struct file * fp,const char __user * ubuf,size_t cnt,loff_t * ppos)184 static ssize_t rsxx_cram_write(struct file *fp, const char __user *ubuf,
185 			       size_t cnt, loff_t *ppos)
186 {
187 	struct rsxx_cardinfo *card = file_inode(fp)->i_private;
188 	char *buf;
189 	ssize_t st;
190 
191 	buf = memdup_user(ubuf, cnt);
192 	if (IS_ERR(buf))
193 		return PTR_ERR(buf);
194 
195 	st = rsxx_creg_write(card, CREG_ADD_CRAM + (u32)*ppos, cnt, buf, 1);
196 	kfree(buf);
197 	if (st)
198 		return st;
199 	*ppos += cnt;
200 	return cnt;
201 }
202 
203 static const struct file_operations debugfs_cram_fops = {
204 	.owner		= THIS_MODULE,
205 	.read		= rsxx_cram_read,
206 	.write		= rsxx_cram_write,
207 };
208 
209 static const struct file_operations debugfs_stats_fops = {
210 	.owner		= THIS_MODULE,
211 	.open		= rsxx_attr_stats_open,
212 	.read		= seq_read,
213 	.llseek		= seq_lseek,
214 	.release	= single_release,
215 };
216 
217 static const struct file_operations debugfs_pci_regs_fops = {
218 	.owner		= THIS_MODULE,
219 	.open		= rsxx_attr_pci_regs_open,
220 	.read		= seq_read,
221 	.llseek		= seq_lseek,
222 	.release	= single_release,
223 };
224 
rsxx_debugfs_dev_new(struct rsxx_cardinfo * card)225 static void rsxx_debugfs_dev_new(struct rsxx_cardinfo *card)
226 {
227 	struct dentry *debugfs_stats;
228 	struct dentry *debugfs_pci_regs;
229 	struct dentry *debugfs_cram;
230 
231 	card->debugfs_dir = debugfs_create_dir(card->gendisk->disk_name, NULL);
232 	if (IS_ERR_OR_NULL(card->debugfs_dir))
233 		goto failed_debugfs_dir;
234 
235 	debugfs_stats = debugfs_create_file("stats", 0444,
236 					    card->debugfs_dir, card,
237 					    &debugfs_stats_fops);
238 	if (IS_ERR_OR_NULL(debugfs_stats))
239 		goto failed_debugfs_stats;
240 
241 	debugfs_pci_regs = debugfs_create_file("pci_regs", 0444,
242 					       card->debugfs_dir, card,
243 					       &debugfs_pci_regs_fops);
244 	if (IS_ERR_OR_NULL(debugfs_pci_regs))
245 		goto failed_debugfs_pci_regs;
246 
247 	debugfs_cram = debugfs_create_file("cram", 0644,
248 					   card->debugfs_dir, card,
249 					   &debugfs_cram_fops);
250 	if (IS_ERR_OR_NULL(debugfs_cram))
251 		goto failed_debugfs_cram;
252 
253 	return;
254 failed_debugfs_cram:
255 	debugfs_remove(debugfs_pci_regs);
256 failed_debugfs_pci_regs:
257 	debugfs_remove(debugfs_stats);
258 failed_debugfs_stats:
259 	debugfs_remove(card->debugfs_dir);
260 failed_debugfs_dir:
261 	card->debugfs_dir = NULL;
262 }
263 
264 /*----------------- Interrupt Control & Handling -------------------*/
265 
rsxx_mask_interrupts(struct rsxx_cardinfo * card)266 static void rsxx_mask_interrupts(struct rsxx_cardinfo *card)
267 {
268 	card->isr_mask = 0;
269 	card->ier_mask = 0;
270 }
271 
__enable_intr(unsigned int * mask,unsigned int intr)272 static void __enable_intr(unsigned int *mask, unsigned int intr)
273 {
274 	*mask |= intr;
275 }
276 
__disable_intr(unsigned int * mask,unsigned int intr)277 static void __disable_intr(unsigned int *mask, unsigned int intr)
278 {
279 	*mask &= ~intr;
280 }
281 
282 /*
283  * NOTE: Disabling the IER will disable the hardware interrupt.
284  * Disabling the ISR will disable the software handling of the ISR bit.
285  *
286  * Enable/Disable interrupt functions assume the card->irq_lock
287  * is held by the caller.
288  */
rsxx_enable_ier(struct rsxx_cardinfo * card,unsigned int intr)289 void rsxx_enable_ier(struct rsxx_cardinfo *card, unsigned int intr)
290 {
291 	if (unlikely(card->halt) ||
292 	    unlikely(card->eeh_state))
293 		return;
294 
295 	__enable_intr(&card->ier_mask, intr);
296 	iowrite32(card->ier_mask, card->regmap + IER);
297 }
298 
rsxx_disable_ier(struct rsxx_cardinfo * card,unsigned int intr)299 void rsxx_disable_ier(struct rsxx_cardinfo *card, unsigned int intr)
300 {
301 	if (unlikely(card->eeh_state))
302 		return;
303 
304 	__disable_intr(&card->ier_mask, intr);
305 	iowrite32(card->ier_mask, card->regmap + IER);
306 }
307 
rsxx_enable_ier_and_isr(struct rsxx_cardinfo * card,unsigned int intr)308 void rsxx_enable_ier_and_isr(struct rsxx_cardinfo *card,
309 				 unsigned int intr)
310 {
311 	if (unlikely(card->halt) ||
312 	    unlikely(card->eeh_state))
313 		return;
314 
315 	__enable_intr(&card->isr_mask, intr);
316 	__enable_intr(&card->ier_mask, intr);
317 	iowrite32(card->ier_mask, card->regmap + IER);
318 }
rsxx_disable_ier_and_isr(struct rsxx_cardinfo * card,unsigned int intr)319 void rsxx_disable_ier_and_isr(struct rsxx_cardinfo *card,
320 				  unsigned int intr)
321 {
322 	if (unlikely(card->eeh_state))
323 		return;
324 
325 	__disable_intr(&card->isr_mask, intr);
326 	__disable_intr(&card->ier_mask, intr);
327 	iowrite32(card->ier_mask, card->regmap + IER);
328 }
329 
rsxx_isr(int irq,void * pdata)330 static irqreturn_t rsxx_isr(int irq, void *pdata)
331 {
332 	struct rsxx_cardinfo *card = pdata;
333 	unsigned int isr;
334 	int handled = 0;
335 	int reread_isr;
336 	int i;
337 
338 	spin_lock(&card->irq_lock);
339 
340 	do {
341 		reread_isr = 0;
342 
343 		if (unlikely(card->eeh_state))
344 			break;
345 
346 		isr = ioread32(card->regmap + ISR);
347 		if (isr == 0xffffffff) {
348 			/*
349 			 * A few systems seem to have an intermittent issue
350 			 * where PCI reads return all Fs, but retrying the read
351 			 * a little later will return as expected.
352 			 */
353 			dev_info(CARD_TO_DEV(card),
354 				"ISR = 0xFFFFFFFF, retrying later\n");
355 			break;
356 		}
357 
358 		isr &= card->isr_mask;
359 		if (!isr)
360 			break;
361 
362 		for (i = 0; i < card->n_targets; i++) {
363 			if (isr & CR_INTR_DMA(i)) {
364 				if (card->ier_mask & CR_INTR_DMA(i)) {
365 					rsxx_disable_ier(card, CR_INTR_DMA(i));
366 					reread_isr = 1;
367 				}
368 				queue_work(card->ctrl[i].done_wq,
369 					   &card->ctrl[i].dma_done_work);
370 				handled++;
371 			}
372 		}
373 
374 		if (isr & CR_INTR_CREG) {
375 			queue_work(card->creg_ctrl.creg_wq,
376 				   &card->creg_ctrl.done_work);
377 			handled++;
378 		}
379 
380 		if (isr & CR_INTR_EVENT) {
381 			queue_work(card->event_wq, &card->event_work);
382 			rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
383 			handled++;
384 		}
385 	} while (reread_isr);
386 
387 	spin_unlock(&card->irq_lock);
388 
389 	return handled ? IRQ_HANDLED : IRQ_NONE;
390 }
391 
392 /*----------------- Card Event Handler -------------------*/
rsxx_card_state_to_str(unsigned int state)393 static const char * const rsxx_card_state_to_str(unsigned int state)
394 {
395 	static const char * const state_strings[] = {
396 		"Unknown", "Shutdown", "Starting", "Formatting",
397 		"Uninitialized", "Good", "Shutting Down",
398 		"Fault", "Read Only Fault", "dStroying"
399 	};
400 
401 	return state_strings[ffs(state)];
402 }
403 
card_state_change(struct rsxx_cardinfo * card,unsigned int new_state)404 static void card_state_change(struct rsxx_cardinfo *card,
405 			      unsigned int new_state)
406 {
407 	int st;
408 
409 	dev_info(CARD_TO_DEV(card),
410 		"card state change detected.(%s -> %s)\n",
411 		rsxx_card_state_to_str(card->state),
412 		rsxx_card_state_to_str(new_state));
413 
414 	card->state = new_state;
415 
416 	/* Don't attach DMA interfaces if the card has an invalid config */
417 	if (!card->config_valid)
418 		return;
419 
420 	switch (new_state) {
421 	case CARD_STATE_RD_ONLY_FAULT:
422 		dev_crit(CARD_TO_DEV(card),
423 			"Hardware has entered read-only mode!\n");
424 		/*
425 		 * Fall through so the DMA devices can be attached and
426 		 * the user can attempt to pull off their data.
427 		 */
428 		/* fall through */
429 	case CARD_STATE_GOOD:
430 		st = rsxx_get_card_size8(card, &card->size8);
431 		if (st)
432 			dev_err(CARD_TO_DEV(card),
433 				"Failed attaching DMA devices\n");
434 
435 		if (card->config_valid)
436 			set_capacity(card->gendisk, card->size8 >> 9);
437 		break;
438 
439 	case CARD_STATE_FAULT:
440 		dev_crit(CARD_TO_DEV(card),
441 			"Hardware Fault reported!\n");
442 		/* Fall through. */
443 
444 	/* Everything else, detach DMA interface if it's attached. */
445 	case CARD_STATE_SHUTDOWN:
446 	case CARD_STATE_STARTING:
447 	case CARD_STATE_FORMATTING:
448 	case CARD_STATE_UNINITIALIZED:
449 	case CARD_STATE_SHUTTING_DOWN:
450 	/*
451 	 * dStroy is a term coined by marketing to represent the low level
452 	 * secure erase.
453 	 */
454 	case CARD_STATE_DSTROYING:
455 		set_capacity(card->gendisk, 0);
456 		break;
457 	}
458 }
459 
card_event_handler(struct work_struct * work)460 static void card_event_handler(struct work_struct *work)
461 {
462 	struct rsxx_cardinfo *card;
463 	unsigned int state;
464 	unsigned long flags;
465 	int st;
466 
467 	card = container_of(work, struct rsxx_cardinfo, event_work);
468 
469 	if (unlikely(card->halt))
470 		return;
471 
472 	/*
473 	 * Enable the interrupt now to avoid any weird race conditions where a
474 	 * state change might occur while rsxx_get_card_state() is
475 	 * processing a returned creg cmd.
476 	 */
477 	spin_lock_irqsave(&card->irq_lock, flags);
478 	rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
479 	spin_unlock_irqrestore(&card->irq_lock, flags);
480 
481 	st = rsxx_get_card_state(card, &state);
482 	if (st) {
483 		dev_info(CARD_TO_DEV(card),
484 			"Failed reading state after event.\n");
485 		return;
486 	}
487 
488 	if (card->state != state)
489 		card_state_change(card, state);
490 
491 	if (card->creg_ctrl.creg_stats.stat & CREG_STAT_LOG_PENDING)
492 		rsxx_read_hw_log(card);
493 }
494 
495 /*----------------- Card Operations -------------------*/
card_shutdown(struct rsxx_cardinfo * card)496 static int card_shutdown(struct rsxx_cardinfo *card)
497 {
498 	unsigned int state;
499 	signed long start;
500 	const int timeout = msecs_to_jiffies(120000);
501 	int st;
502 
503 	/* We can't issue a shutdown if the card is in a transition state */
504 	start = jiffies;
505 	do {
506 		st = rsxx_get_card_state(card, &state);
507 		if (st)
508 			return st;
509 	} while (state == CARD_STATE_STARTING &&
510 		 (jiffies - start < timeout));
511 
512 	if (state == CARD_STATE_STARTING)
513 		return -ETIMEDOUT;
514 
515 	/* Only issue a shutdown if we need to */
516 	if ((state != CARD_STATE_SHUTTING_DOWN) &&
517 	    (state != CARD_STATE_SHUTDOWN)) {
518 		st = rsxx_issue_card_cmd(card, CARD_CMD_SHUTDOWN);
519 		if (st)
520 			return st;
521 	}
522 
523 	start = jiffies;
524 	do {
525 		st = rsxx_get_card_state(card, &state);
526 		if (st)
527 			return st;
528 	} while (state != CARD_STATE_SHUTDOWN &&
529 		 (jiffies - start < timeout));
530 
531 	if (state != CARD_STATE_SHUTDOWN)
532 		return -ETIMEDOUT;
533 
534 	return 0;
535 }
536 
rsxx_eeh_frozen(struct pci_dev * dev)537 static int rsxx_eeh_frozen(struct pci_dev *dev)
538 {
539 	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
540 	int i;
541 	int st;
542 
543 	dev_warn(&dev->dev, "IBM Flash Adapter PCI: preparing for slot reset.\n");
544 
545 	card->eeh_state = 1;
546 	rsxx_mask_interrupts(card);
547 
548 	/*
549 	 * We need to guarantee that the write for eeh_state and masking
550 	 * interrupts does not become reordered. This will prevent a possible
551 	 * race condition with the EEH code.
552 	 */
553 	wmb();
554 
555 	pci_disable_device(dev);
556 
557 	st = rsxx_eeh_save_issued_dmas(card);
558 	if (st)
559 		return st;
560 
561 	rsxx_eeh_save_issued_creg(card);
562 
563 	for (i = 0; i < card->n_targets; i++) {
564 		if (card->ctrl[i].status.buf)
565 			pci_free_consistent(card->dev, STATUS_BUFFER_SIZE8,
566 					    card->ctrl[i].status.buf,
567 					    card->ctrl[i].status.dma_addr);
568 		if (card->ctrl[i].cmd.buf)
569 			pci_free_consistent(card->dev, COMMAND_BUFFER_SIZE8,
570 					    card->ctrl[i].cmd.buf,
571 					    card->ctrl[i].cmd.dma_addr);
572 	}
573 
574 	return 0;
575 }
576 
rsxx_eeh_failure(struct pci_dev * dev)577 static void rsxx_eeh_failure(struct pci_dev *dev)
578 {
579 	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
580 	int i;
581 	int cnt = 0;
582 
583 	dev_err(&dev->dev, "IBM Flash Adapter PCI: disabling failed card.\n");
584 
585 	card->eeh_state = 1;
586 	card->halt = 1;
587 
588 	for (i = 0; i < card->n_targets; i++) {
589 		spin_lock_bh(&card->ctrl[i].queue_lock);
590 		cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
591 					     &card->ctrl[i].queue,
592 					     COMPLETE_DMA);
593 		spin_unlock_bh(&card->ctrl[i].queue_lock);
594 
595 		cnt += rsxx_dma_cancel(&card->ctrl[i]);
596 
597 		if (cnt)
598 			dev_info(CARD_TO_DEV(card),
599 				"Freed %d queued DMAs on channel %d\n",
600 				cnt, card->ctrl[i].id);
601 	}
602 }
603 
rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo * card)604 static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
605 {
606 	unsigned int status;
607 	int iter = 0;
608 
609 	/* We need to wait for the hardware to reset */
610 	while (iter++ < 10) {
611 		status = ioread32(card->regmap + PCI_RECONFIG);
612 
613 		if (status & RSXX_FLUSH_BUSY) {
614 			ssleep(1);
615 			continue;
616 		}
617 
618 		if (status & RSXX_FLUSH_TIMEOUT)
619 			dev_warn(CARD_TO_DEV(card), "HW: flash controller timeout\n");
620 		return 0;
621 	}
622 
623 	/* Hardware failed resetting itself. */
624 	return -1;
625 }
626 
rsxx_error_detected(struct pci_dev * dev,enum pci_channel_state error)627 static pci_ers_result_t rsxx_error_detected(struct pci_dev *dev,
628 					    enum pci_channel_state error)
629 {
630 	int st;
631 
632 	if (dev->revision < RSXX_EEH_SUPPORT)
633 		return PCI_ERS_RESULT_NONE;
634 
635 	if (error == pci_channel_io_perm_failure) {
636 		rsxx_eeh_failure(dev);
637 		return PCI_ERS_RESULT_DISCONNECT;
638 	}
639 
640 	st = rsxx_eeh_frozen(dev);
641 	if (st) {
642 		dev_err(&dev->dev, "Slot reset setup failed\n");
643 		rsxx_eeh_failure(dev);
644 		return PCI_ERS_RESULT_DISCONNECT;
645 	}
646 
647 	return PCI_ERS_RESULT_NEED_RESET;
648 }
649 
rsxx_slot_reset(struct pci_dev * dev)650 static pci_ers_result_t rsxx_slot_reset(struct pci_dev *dev)
651 {
652 	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
653 	unsigned long flags;
654 	int i;
655 	int st;
656 
657 	dev_warn(&dev->dev,
658 		"IBM Flash Adapter PCI: recovering from slot reset.\n");
659 
660 	st = pci_enable_device(dev);
661 	if (st)
662 		goto failed_hw_setup;
663 
664 	pci_set_master(dev);
665 
666 	st = rsxx_eeh_fifo_flush_poll(card);
667 	if (st)
668 		goto failed_hw_setup;
669 
670 	rsxx_dma_queue_reset(card);
671 
672 	for (i = 0; i < card->n_targets; i++) {
673 		st = rsxx_hw_buffers_init(dev, &card->ctrl[i]);
674 		if (st)
675 			goto failed_hw_buffers_init;
676 	}
677 
678 	if (card->config_valid)
679 		rsxx_dma_configure(card);
680 
681 	/* Clears the ISR register from spurious interrupts */
682 	st = ioread32(card->regmap + ISR);
683 
684 	card->eeh_state = 0;
685 
686 	spin_lock_irqsave(&card->irq_lock, flags);
687 	if (card->n_targets & RSXX_MAX_TARGETS)
688 		rsxx_enable_ier_and_isr(card, CR_INTR_ALL_G);
689 	else
690 		rsxx_enable_ier_and_isr(card, CR_INTR_ALL_C);
691 	spin_unlock_irqrestore(&card->irq_lock, flags);
692 
693 	rsxx_kick_creg_queue(card);
694 
695 	for (i = 0; i < card->n_targets; i++) {
696 		spin_lock(&card->ctrl[i].queue_lock);
697 		if (list_empty(&card->ctrl[i].queue)) {
698 			spin_unlock(&card->ctrl[i].queue_lock);
699 			continue;
700 		}
701 		spin_unlock(&card->ctrl[i].queue_lock);
702 
703 		queue_work(card->ctrl[i].issue_wq,
704 				&card->ctrl[i].issue_dma_work);
705 	}
706 
707 	dev_info(&dev->dev, "IBM Flash Adapter PCI: recovery complete.\n");
708 
709 	return PCI_ERS_RESULT_RECOVERED;
710 
711 failed_hw_buffers_init:
712 	for (i = 0; i < card->n_targets; i++) {
713 		if (card->ctrl[i].status.buf)
714 			pci_free_consistent(card->dev,
715 					STATUS_BUFFER_SIZE8,
716 					card->ctrl[i].status.buf,
717 					card->ctrl[i].status.dma_addr);
718 		if (card->ctrl[i].cmd.buf)
719 			pci_free_consistent(card->dev,
720 					COMMAND_BUFFER_SIZE8,
721 					card->ctrl[i].cmd.buf,
722 					card->ctrl[i].cmd.dma_addr);
723 	}
724 failed_hw_setup:
725 	rsxx_eeh_failure(dev);
726 	return PCI_ERS_RESULT_DISCONNECT;
727 
728 }
729 
730 /*----------------- Driver Initialization & Setup -------------------*/
731 /* Returns:   0 if the driver is compatible with the device
732 	     -1 if the driver is NOT compatible with the device */
rsxx_compatibility_check(struct rsxx_cardinfo * card)733 static int rsxx_compatibility_check(struct rsxx_cardinfo *card)
734 {
735 	unsigned char pci_rev;
736 
737 	pci_read_config_byte(card->dev, PCI_REVISION_ID, &pci_rev);
738 
739 	if (pci_rev > RS70_PCI_REV_SUPPORTED)
740 		return -1;
741 	return 0;
742 }
743 
rsxx_pci_probe(struct pci_dev * dev,const struct pci_device_id * id)744 static int rsxx_pci_probe(struct pci_dev *dev,
745 					const struct pci_device_id *id)
746 {
747 	struct rsxx_cardinfo *card;
748 	int st;
749 	unsigned int sync_timeout;
750 
751 	dev_info(&dev->dev, "PCI-Flash SSD discovered\n");
752 
753 	card = kzalloc(sizeof(*card), GFP_KERNEL);
754 	if (!card)
755 		return -ENOMEM;
756 
757 	card->dev = dev;
758 	pci_set_drvdata(dev, card);
759 
760 	st = ida_alloc(&rsxx_disk_ida, GFP_KERNEL);
761 	if (st < 0)
762 		goto failed_ida_get;
763 	card->disk_id = st;
764 
765 	st = pci_enable_device(dev);
766 	if (st)
767 		goto failed_enable;
768 
769 	pci_set_master(dev);
770 
771 	st = dma_set_mask(&dev->dev, DMA_BIT_MASK(64));
772 	if (st) {
773 		dev_err(CARD_TO_DEV(card),
774 			"No usable DMA configuration,aborting\n");
775 		goto failed_dma_mask;
776 	}
777 
778 	st = pci_request_regions(dev, DRIVER_NAME);
779 	if (st) {
780 		dev_err(CARD_TO_DEV(card),
781 			"Failed to request memory region\n");
782 		goto failed_request_regions;
783 	}
784 
785 	if (pci_resource_len(dev, 0) == 0) {
786 		dev_err(CARD_TO_DEV(card), "BAR0 has length 0!\n");
787 		st = -ENOMEM;
788 		goto failed_iomap;
789 	}
790 
791 	card->regmap = pci_iomap(dev, 0, 0);
792 	if (!card->regmap) {
793 		dev_err(CARD_TO_DEV(card), "Failed to map BAR0\n");
794 		st = -ENOMEM;
795 		goto failed_iomap;
796 	}
797 
798 	spin_lock_init(&card->irq_lock);
799 	card->halt = 0;
800 	card->eeh_state = 0;
801 
802 	spin_lock_irq(&card->irq_lock);
803 	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
804 	spin_unlock_irq(&card->irq_lock);
805 
806 	if (!force_legacy) {
807 		st = pci_enable_msi(dev);
808 		if (st)
809 			dev_warn(CARD_TO_DEV(card),
810 				"Failed to enable MSI\n");
811 	}
812 
813 	st = request_irq(dev->irq, rsxx_isr, IRQF_SHARED,
814 			 DRIVER_NAME, card);
815 	if (st) {
816 		dev_err(CARD_TO_DEV(card),
817 			"Failed requesting IRQ%d\n", dev->irq);
818 		goto failed_irq;
819 	}
820 
821 	/************* Setup Processor Command Interface *************/
822 	st = rsxx_creg_setup(card);
823 	if (st) {
824 		dev_err(CARD_TO_DEV(card), "Failed to setup creg interface.\n");
825 		goto failed_creg_setup;
826 	}
827 
828 	spin_lock_irq(&card->irq_lock);
829 	rsxx_enable_ier_and_isr(card, CR_INTR_CREG);
830 	spin_unlock_irq(&card->irq_lock);
831 
832 	st = rsxx_compatibility_check(card);
833 	if (st) {
834 		dev_warn(CARD_TO_DEV(card),
835 			"Incompatible driver detected. Please update the driver.\n");
836 		st = -EINVAL;
837 		goto failed_compatiblity_check;
838 	}
839 
840 	/************* Load Card Config *************/
841 	st = rsxx_load_config(card);
842 	if (st)
843 		dev_err(CARD_TO_DEV(card),
844 			"Failed loading card config\n");
845 
846 	/************* Setup DMA Engine *************/
847 	st = rsxx_get_num_targets(card, &card->n_targets);
848 	if (st)
849 		dev_info(CARD_TO_DEV(card),
850 			"Failed reading the number of DMA targets\n");
851 
852 	card->ctrl = kcalloc(card->n_targets, sizeof(*card->ctrl),
853 			     GFP_KERNEL);
854 	if (!card->ctrl) {
855 		st = -ENOMEM;
856 		goto failed_dma_setup;
857 	}
858 
859 	st = rsxx_dma_setup(card);
860 	if (st) {
861 		dev_info(CARD_TO_DEV(card),
862 			"Failed to setup DMA engine\n");
863 		goto failed_dma_setup;
864 	}
865 
866 	/************* Setup Card Event Handler *************/
867 	card->event_wq = create_singlethread_workqueue(DRIVER_NAME"_event");
868 	if (!card->event_wq) {
869 		dev_err(CARD_TO_DEV(card), "Failed card event setup.\n");
870 		goto failed_event_handler;
871 	}
872 
873 	INIT_WORK(&card->event_work, card_event_handler);
874 
875 	st = rsxx_setup_dev(card);
876 	if (st)
877 		goto failed_create_dev;
878 
879 	rsxx_get_card_state(card, &card->state);
880 
881 	dev_info(CARD_TO_DEV(card),
882 		"card state: %s\n",
883 		rsxx_card_state_to_str(card->state));
884 
885 	/*
886 	 * Now that the DMA Engine and devices have been setup,
887 	 * we can enable the event interrupt(it kicks off actions in
888 	 * those layers so we couldn't enable it right away.)
889 	 */
890 	spin_lock_irq(&card->irq_lock);
891 	rsxx_enable_ier_and_isr(card, CR_INTR_EVENT);
892 	spin_unlock_irq(&card->irq_lock);
893 
894 	if (card->state == CARD_STATE_SHUTDOWN) {
895 		st = rsxx_issue_card_cmd(card, CARD_CMD_STARTUP);
896 		if (st)
897 			dev_crit(CARD_TO_DEV(card),
898 				"Failed issuing card startup\n");
899 		if (sync_start) {
900 			sync_timeout = SYNC_START_TIMEOUT;
901 
902 			dev_info(CARD_TO_DEV(card),
903 				 "Waiting for card to startup\n");
904 
905 			do {
906 				ssleep(1);
907 				sync_timeout--;
908 
909 				rsxx_get_card_state(card, &card->state);
910 			} while (sync_timeout &&
911 				(card->state == CARD_STATE_STARTING));
912 
913 			if (card->state == CARD_STATE_STARTING) {
914 				dev_warn(CARD_TO_DEV(card),
915 					 "Card startup timed out\n");
916 				card->size8 = 0;
917 			} else {
918 				dev_info(CARD_TO_DEV(card),
919 					"card state: %s\n",
920 					rsxx_card_state_to_str(card->state));
921 				st = rsxx_get_card_size8(card, &card->size8);
922 				if (st)
923 					card->size8 = 0;
924 			}
925 		}
926 	} else if (card->state == CARD_STATE_GOOD ||
927 		   card->state == CARD_STATE_RD_ONLY_FAULT) {
928 		st = rsxx_get_card_size8(card, &card->size8);
929 		if (st)
930 			card->size8 = 0;
931 	}
932 
933 	rsxx_attach_dev(card);
934 
935 	/************* Setup Debugfs *************/
936 	rsxx_debugfs_dev_new(card);
937 
938 	return 0;
939 
940 failed_create_dev:
941 	destroy_workqueue(card->event_wq);
942 	card->event_wq = NULL;
943 failed_event_handler:
944 	rsxx_dma_destroy(card);
945 failed_dma_setup:
946 failed_compatiblity_check:
947 	destroy_workqueue(card->creg_ctrl.creg_wq);
948 	card->creg_ctrl.creg_wq = NULL;
949 failed_creg_setup:
950 	spin_lock_irq(&card->irq_lock);
951 	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
952 	spin_unlock_irq(&card->irq_lock);
953 	free_irq(dev->irq, card);
954 	if (!force_legacy)
955 		pci_disable_msi(dev);
956 failed_irq:
957 	pci_iounmap(dev, card->regmap);
958 failed_iomap:
959 	pci_release_regions(dev);
960 failed_request_regions:
961 failed_dma_mask:
962 	pci_disable_device(dev);
963 failed_enable:
964 	ida_free(&rsxx_disk_ida, card->disk_id);
965 failed_ida_get:
966 	kfree(card);
967 
968 	return st;
969 }
970 
rsxx_pci_remove(struct pci_dev * dev)971 static void rsxx_pci_remove(struct pci_dev *dev)
972 {
973 	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
974 	unsigned long flags;
975 	int st;
976 	int i;
977 
978 	if (!card)
979 		return;
980 
981 	dev_info(CARD_TO_DEV(card),
982 		"Removing PCI-Flash SSD.\n");
983 
984 	rsxx_detach_dev(card);
985 
986 	for (i = 0; i < card->n_targets; i++) {
987 		spin_lock_irqsave(&card->irq_lock, flags);
988 		rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
989 		spin_unlock_irqrestore(&card->irq_lock, flags);
990 	}
991 
992 	st = card_shutdown(card);
993 	if (st)
994 		dev_crit(CARD_TO_DEV(card), "Shutdown failed!\n");
995 
996 	/* Sync outstanding event handlers. */
997 	spin_lock_irqsave(&card->irq_lock, flags);
998 	rsxx_disable_ier_and_isr(card, CR_INTR_EVENT);
999 	spin_unlock_irqrestore(&card->irq_lock, flags);
1000 
1001 	cancel_work_sync(&card->event_work);
1002 
1003 	destroy_workqueue(card->event_wq);
1004 	rsxx_destroy_dev(card);
1005 	rsxx_dma_destroy(card);
1006 	destroy_workqueue(card->creg_ctrl.creg_wq);
1007 
1008 	spin_lock_irqsave(&card->irq_lock, flags);
1009 	rsxx_disable_ier_and_isr(card, CR_INTR_ALL);
1010 	spin_unlock_irqrestore(&card->irq_lock, flags);
1011 
1012 	/* Prevent work_structs from re-queuing themselves. */
1013 	card->halt = 1;
1014 
1015 	debugfs_remove_recursive(card->debugfs_dir);
1016 
1017 	free_irq(dev->irq, card);
1018 
1019 	if (!force_legacy)
1020 		pci_disable_msi(dev);
1021 
1022 	rsxx_creg_destroy(card);
1023 
1024 	pci_iounmap(dev, card->regmap);
1025 
1026 	pci_disable_device(dev);
1027 	pci_release_regions(dev);
1028 
1029 	ida_free(&rsxx_disk_ida, card->disk_id);
1030 	kfree(card);
1031 }
1032 
rsxx_pci_suspend(struct pci_dev * dev,pm_message_t state)1033 static int rsxx_pci_suspend(struct pci_dev *dev, pm_message_t state)
1034 {
1035 	/* We don't support suspend at this time. */
1036 	return -ENOSYS;
1037 }
1038 
rsxx_pci_shutdown(struct pci_dev * dev)1039 static void rsxx_pci_shutdown(struct pci_dev *dev)
1040 {
1041 	struct rsxx_cardinfo *card = pci_get_drvdata(dev);
1042 	unsigned long flags;
1043 	int i;
1044 
1045 	if (!card)
1046 		return;
1047 
1048 	dev_info(CARD_TO_DEV(card), "Shutting down PCI-Flash SSD.\n");
1049 
1050 	rsxx_detach_dev(card);
1051 
1052 	for (i = 0; i < card->n_targets; i++) {
1053 		spin_lock_irqsave(&card->irq_lock, flags);
1054 		rsxx_disable_ier_and_isr(card, CR_INTR_DMA(i));
1055 		spin_unlock_irqrestore(&card->irq_lock, flags);
1056 	}
1057 
1058 	card_shutdown(card);
1059 }
1060 
1061 static const struct pci_error_handlers rsxx_err_handler = {
1062 	.error_detected = rsxx_error_detected,
1063 	.slot_reset     = rsxx_slot_reset,
1064 };
1065 
1066 static const struct pci_device_id rsxx_pci_ids[] = {
1067 	{PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS70_FLASH)},
1068 	{PCI_DEVICE(PCI_VENDOR_ID_IBM, PCI_DEVICE_ID_FS80_FLASH)},
1069 	{0,},
1070 };
1071 
1072 MODULE_DEVICE_TABLE(pci, rsxx_pci_ids);
1073 
1074 static struct pci_driver rsxx_pci_driver = {
1075 	.name		= DRIVER_NAME,
1076 	.id_table	= rsxx_pci_ids,
1077 	.probe		= rsxx_pci_probe,
1078 	.remove		= rsxx_pci_remove,
1079 	.suspend	= rsxx_pci_suspend,
1080 	.shutdown	= rsxx_pci_shutdown,
1081 	.err_handler    = &rsxx_err_handler,
1082 };
1083 
rsxx_core_init(void)1084 static int __init rsxx_core_init(void)
1085 {
1086 	int st;
1087 
1088 	st = rsxx_dev_init();
1089 	if (st)
1090 		return st;
1091 
1092 	st = rsxx_dma_init();
1093 	if (st)
1094 		goto dma_init_failed;
1095 
1096 	st = rsxx_creg_init();
1097 	if (st)
1098 		goto creg_init_failed;
1099 
1100 	return pci_register_driver(&rsxx_pci_driver);
1101 
1102 creg_init_failed:
1103 	rsxx_dma_cleanup();
1104 dma_init_failed:
1105 	rsxx_dev_cleanup();
1106 
1107 	return st;
1108 }
1109 
rsxx_core_cleanup(void)1110 static void __exit rsxx_core_cleanup(void)
1111 {
1112 	pci_unregister_driver(&rsxx_pci_driver);
1113 	rsxx_creg_cleanup();
1114 	rsxx_dma_cleanup();
1115 	rsxx_dev_cleanup();
1116 }
1117 
1118 module_init(rsxx_core_init);
1119 module_exit(rsxx_core_cleanup);
1120