62 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
64 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info);
65 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
67 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
70 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
73 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
201 static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_offline()  argument
203 	return !ctrl_info->controller_online;  in pqi_ctrl_offline()
206 static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)  in pqi_check_ctrl_health()  argument
208 	if (ctrl_info->controller_online)  in pqi_check_ctrl_health()
209 		if (!sis_is_firmware_running(ctrl_info))  in pqi_check_ctrl_health()
210 			pqi_take_ctrl_offline(ctrl_info);  in pqi_check_ctrl_health()
219 	struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_mode()  argument
221 	return sis_read_driver_scratch(ctrl_info);  in pqi_get_ctrl_mode()
224 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_save_ctrl_mode()  argument
227 	sis_write_driver_scratch(ctrl_info, mode);  in pqi_save_ctrl_mode()
230 static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_requests()  argument
232 	ctrl_info->block_requests = true;  in pqi_ctrl_block_requests()
233 	scsi_block_requests(ctrl_info->scsi_host);  in pqi_ctrl_block_requests()
236 static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_requests()  argument
238 	ctrl_info->block_requests = false;  in pqi_ctrl_unblock_requests()
239 	wake_up_all(&ctrl_info->block_requests_wait);  in pqi_ctrl_unblock_requests()
240 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_ctrl_unblock_requests()
241 	scsi_unblock_requests(ctrl_info->scsi_host);  in pqi_ctrl_unblock_requests()
244 static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_blocked()  argument
246 	return ctrl_info->block_requests;  in pqi_ctrl_blocked()
249 static unsigned long pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_if_ctrl_blocked()  argument
254 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_wait_if_ctrl_blocked()
257 	atomic_inc(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
260 		wait_event(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
261 			!pqi_ctrl_blocked(ctrl_info));  in pqi_wait_if_ctrl_blocked()
267 			wait_event_timeout(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
268 				!pqi_ctrl_blocked(ctrl_info),  in pqi_wait_if_ctrl_blocked()
273 	atomic_dec(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
278 static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_busy()  argument
280 	atomic_inc(&ctrl_info->num_busy_threads);  in pqi_ctrl_busy()
283 static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unbusy()  argument
285 	atomic_dec(&ctrl_info->num_busy_threads);  in pqi_ctrl_unbusy()
288 static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_until_quiesced()  argument
290 	while (atomic_read(&ctrl_info->num_busy_threads) >  in pqi_ctrl_wait_until_quiesced()
291 		atomic_read(&ctrl_info->num_blocked_threads))  in pqi_ctrl_wait_until_quiesced()
316 	struct pqi_ctrl_info *ctrl_info, unsigned long delay)  in pqi_schedule_rescan_worker_with_delay()  argument
318 	if (pqi_ctrl_offline(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
321 	schedule_delayed_work(&ctrl_info->rescan_work, delay);  in pqi_schedule_rescan_worker_with_delay()
324 static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker()  argument
326 	pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);  in pqi_schedule_rescan_worker()
332 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker_delayed()  argument
334 	pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);  in pqi_schedule_rescan_worker_delayed()
337 static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_rescan_worker()  argument
339 	cancel_delayed_work_sync(&ctrl_info->rescan_work);  in pqi_cancel_rescan_worker()
342 static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)  in pqi_read_heartbeat_counter()  argument
344 	if (!ctrl_info->heartbeat_counter)  in pqi_read_heartbeat_counter()
347 	return readl(ctrl_info->heartbeat_counter);  in pqi_read_heartbeat_counter()
387 static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_path_request()  argument
454 		dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n",  in pqi_build_raid_path_request()
476 	return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],  in pqi_build_raid_path_request()
489 	struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_request()  argument
492 	u16 i = ctrl_info->next_io_request_slot;	/* benignly racy */  in pqi_alloc_io_request()
495 		io_request = &ctrl_info->io_request_pool[i];  in pqi_alloc_io_request()
499 		i = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
503 	ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
515 static int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_controller()  argument
522 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_identify_controller()
528 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_identify_controller()
531 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_identify_controller()
537 static int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_inquiry()  argument
544 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_scsi_inquiry()
550 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_scsi_inquiry()
553 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_scsi_inquiry()
559 static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_physical_device()  argument
569 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_identify_physical_device()
579 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_identify_physical_device()
582 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_identify_physical_device()
588 static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,  in pqi_flush_cache()  argument
600 	if (pqi_ctrl_offline(ctrl_info))  in pqi_flush_cache()
609 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_flush_cache()
615 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_flush_cache()
618 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_flush_cache()
627 static int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,  in pqi_write_host_wellness()  argument
634 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_write_host_wellness()
640 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_write_host_wellness()
643 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_write_host_wellness()
662 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_driver_version_to_host_wellness()  argument
688 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_driver_version_to_host_wellness()
709 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_current_time_to_host_wellness()  argument
751 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_current_time_to_host_wellness()
763 	struct pqi_ctrl_info *ctrl_info;  in pqi_update_time_worker()  local
765 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_update_time_worker()
768 	if (pqi_ctrl_offline(ctrl_info))  in pqi_update_time_worker()
771 	rc = pqi_write_current_time_to_host_wellness(ctrl_info);  in pqi_update_time_worker()
773 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_time_worker()
776 	schedule_delayed_work(&ctrl_info->update_time_work,  in pqi_update_time_worker()
781 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_update_time_worker()  argument
783 	schedule_delayed_work(&ctrl_info->update_time_work, 0);  in pqi_schedule_update_time_worker()
787 	struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_update_time_worker()  argument
789 	cancel_delayed_work_sync(&ctrl_info->update_time_work);  in pqi_cancel_update_time_worker()
792 static int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_luns()  argument
799 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_report_luns()
804 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_report_luns()
807 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_report_luns()
813 static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_phys_logical_luns()  argument
829 	rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,  in pqi_report_phys_logical_luns()
850 	rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);  in pqi_report_phys_logical_luns()
876 static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_phys_luns()  argument
879 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,  in pqi_report_phys_luns()
883 static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_logical_luns()  argument
886 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);  in pqi_report_logical_luns()
889 static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_lists()  argument
900 	rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);  in pqi_get_device_lists()
902 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
905 	rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);  in pqi_get_device_lists()
907 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
998 static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_level()  argument
1009 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_level()
1022 static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_validate_raid_map()  argument
1080 	dev_warn(&ctrl_info->pci_dev->dev,  in pqi_validate_raid_map()
1088 static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_map()  argument
1100 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_get_raid_map()
1106 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_get_raid_map()
1109 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_get_raid_map()
1115 	rc = pqi_validate_raid_map(ctrl_info, device, raid_map);  in pqi_get_raid_map()
1129 static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_bypass_status()  argument
1140 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_bypass_status()
1154 		pqi_get_raid_map(ctrl_info, device) == 0)  in pqi_get_raid_bypass_status()
1165 static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_volume_status()  argument
1179 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_volume_status()
1202 static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_info()  argument
1215 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0,  in pqi_get_device_info()
1238 			pqi_get_raid_level(ctrl_info, device);  in pqi_get_device_info()
1239 			pqi_get_raid_bypass_status(ctrl_info, device);  in pqi_get_device_info()
1240 			pqi_get_volume_status(ctrl_info, device);  in pqi_get_device_info()
1250 static void pqi_get_physical_disk_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_physical_disk_info()  argument
1258 	rc = pqi_identify_physical_device(ctrl_info, device,  in pqi_get_physical_disk_info()
1279 static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_show_volume_status()  argument
1376 	dev_info(&ctrl_info->pci_dev->dev,  in pqi_show_volume_status()
1378 		ctrl_info->scsi_host->host_no,  in pqi_show_volume_status()
1384 	struct pqi_ctrl_info *ctrl_info;  in pqi_rescan_worker()  local
1386 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_rescan_worker()
1389 	pqi_scan_scsi_devices(ctrl_info);  in pqi_rescan_worker()
1392 static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,  in pqi_add_device()  argument
1398 		rc = scsi_add_device(ctrl_info->scsi_host, device->bus,  in pqi_add_device()
1401 		rc = pqi_add_sas_device(ctrl_info->sas_host, device);  in pqi_add_device()
1406 static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,  in pqi_remove_device()  argument
1417 static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,  in pqi_find_scsi_dev()  argument
1422 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_find_scsi_dev()
1450 static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_find_entry()  argument
1456 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_scsi_find_entry()
1475 static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,  in pqi_dev_info()  argument
1482 		"%d:%d:", ctrl_info->scsi_host->host_no, device->bus);  in pqi_dev_info()
1531 	dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);  in pqi_dev_info()
1595 static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,  in pqi_fixup_botched_add()  argument
1600 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1602 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1608 static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,  in pqi_update_device_list()  argument
1628 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1631 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1638 		find_result = pqi_scsi_find_entry(ctrl_info, device,  in pqi_update_device_list()
1669 	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1685 			&ctrl_info->scsi_device_list);  in pqi_update_device_list()
1691 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1697 			pqi_dev_info(ctrl_info, "offline", device);  in pqi_update_device_list()
1698 			pqi_show_volume_status(ctrl_info, device);  in pqi_update_device_list()
1700 			pqi_dev_info(ctrl_info, "removed", device);  in pqi_update_device_list()
1703 			pqi_remove_device(ctrl_info, device);  in pqi_update_device_list()
1712 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1725 			pqi_dev_info(ctrl_info, "added", device);  in pqi_update_device_list()
1726 			rc = pqi_add_device(ctrl_info, device);  in pqi_update_device_list()
1728 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_device_list()
1730 					ctrl_info->scsi_host->host_no,  in pqi_update_device_list()
1733 				pqi_fixup_botched_add(ctrl_info, device);  in pqi_update_device_list()
1775 static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_update_scsi_devices()  argument
1797 	rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);  in pqi_update_scsi_devices()
1824 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
1837 		dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);  in pqi_update_scsi_devices()
1845 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
1888 		rc = pqi_get_device_info(ctrl_info, device);  in pqi_update_scsi_devices()
1890 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
1896 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
1901 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
1936 					pqi_get_physical_disk_info(ctrl_info,  in pqi_update_scsi_devices()
1946 	pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);  in pqi_update_scsi_devices()
1965 static void pqi_remove_all_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_all_scsi_devices()  argument
1971 		spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_remove_all_scsi_devices()
1973 		device = list_first_entry_or_null(&ctrl_info->scsi_device_list,  in pqi_remove_all_scsi_devices()
1978 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_remove_all_scsi_devices()
1985 			pqi_remove_device(ctrl_info, device);  in pqi_remove_all_scsi_devices()
1990 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_scan_scsi_devices()  argument
1994 	if (pqi_ctrl_offline(ctrl_info))  in pqi_scan_scsi_devices()
1997 	mutex_lock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
1999 	rc = pqi_update_scsi_devices(ctrl_info);  in pqi_scan_scsi_devices()
2001 		pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2003 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
2018 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_finished()  local
2020 	ctrl_info = shost_priv(shost);  in pqi_scan_finished()
2022 	return !mutex_is_locked(&ctrl_info->scan_mutex);  in pqi_scan_finished()
2025 static void pqi_wait_until_scan_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_scan_finished()  argument
2027 	mutex_lock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2028 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2031 static void pqi_wait_until_lun_reset_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_lun_reset_finished()  argument
2033 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2034 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2064 static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_bypass_submit_scsi_cmd()  argument
2390 	return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,  in pqi_raid_bypass_submit_scsi_cmd()
2408 static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_mode_ready()  argument
2415 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_mode_ready()
2424 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2436 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2448 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2469 	struct pqi_ctrl_info *ctrl_info;  in pqi_take_device_offline()  local
2478 	ctrl_info = shost_to_hba(sdev->host);  in pqi_take_device_offline()
2479 	pqi_schedule_rescan_worker(ctrl_info);  in pqi_take_device_offline()
2480 	dev_err(&ctrl_info->pci_dev->dev, "offlined %s scsi %d:%d:%d:%d\n",  in pqi_take_device_offline()
2481 		path, ctrl_info->scsi_host->host_no, device->bus,  in pqi_take_device_offline()
2692 static unsigned int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info,  in pqi_process_io_intr()  argument
2715 		WARN_ON(request_id >= ctrl_info->max_io_slots);  in pqi_process_io_intr()
2717 		io_request = &ctrl_info->io_request_pool[request_id];  in pqi_process_io_intr()
2736 			io_request->error_info = ctrl_info->error_buffer +  in pqi_process_io_intr()
2743 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
2757 		oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;  in pqi_process_io_intr()
2781 static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,  in pqi_send_event_ack()  argument
2790 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
2800 			ctrl_info->num_elements_per_iq))  in pqi_send_event_ack()
2806 		if (pqi_ctrl_offline(ctrl_info))  in pqi_send_event_ack()
2815 	iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;  in pqi_send_event_ack()
2827 static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,  in pqi_acknowledge_event()  argument
2841 	pqi_send_event_ack(ctrl_info, &request, sizeof(request));  in pqi_acknowledge_event()
2847 	struct pqi_ctrl_info *ctrl_info;  in pqi_event_worker()  local
2850 	ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);  in pqi_event_worker()
2852 	pqi_ctrl_busy(ctrl_info);  in pqi_event_worker()
2853 	pqi_wait_if_ctrl_blocked(ctrl_info, NO_TIMEOUT);  in pqi_event_worker()
2854 	if (pqi_ctrl_offline(ctrl_info))  in pqi_event_worker()
2857 	pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_event_worker()
2859 	event = ctrl_info->events;  in pqi_event_worker()
2863 			pqi_acknowledge_event(ctrl_info, event);  in pqi_event_worker()
2869 	pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
2878 	struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t,  in pqi_heartbeat_timer_handler()  local
2881 	pqi_check_ctrl_health(ctrl_info);  in pqi_heartbeat_timer_handler()
2882 	if (pqi_ctrl_offline(ctrl_info))  in pqi_heartbeat_timer_handler()
2885 	num_interrupts = atomic_read(&ctrl_info->num_interrupts);  in pqi_heartbeat_timer_handler()
2886 	heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);  in pqi_heartbeat_timer_handler()
2888 	if (num_interrupts == ctrl_info->previous_num_interrupts) {  in pqi_heartbeat_timer_handler()
2889 		if (heartbeat_count == ctrl_info->previous_heartbeat_count) {  in pqi_heartbeat_timer_handler()
2890 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_heartbeat_timer_handler()
2893 			pqi_take_ctrl_offline(ctrl_info);  in pqi_heartbeat_timer_handler()
2897 		ctrl_info->previous_num_interrupts = num_interrupts;  in pqi_heartbeat_timer_handler()
2900 	ctrl_info->previous_heartbeat_count = heartbeat_count;  in pqi_heartbeat_timer_handler()
2901 	mod_timer(&ctrl_info->heartbeat_timer,  in pqi_heartbeat_timer_handler()
2905 static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_start_heartbeat_timer()  argument
2907 	if (!ctrl_info->heartbeat_counter)  in pqi_start_heartbeat_timer()
2910 	ctrl_info->previous_num_interrupts =  in pqi_start_heartbeat_timer()
2911 		atomic_read(&ctrl_info->num_interrupts);  in pqi_start_heartbeat_timer()
2912 	ctrl_info->previous_heartbeat_count =  in pqi_start_heartbeat_timer()
2913 		pqi_read_heartbeat_counter(ctrl_info);  in pqi_start_heartbeat_timer()
2915 	ctrl_info->heartbeat_timer.expires =  in pqi_start_heartbeat_timer()
2917 	add_timer(&ctrl_info->heartbeat_timer);  in pqi_start_heartbeat_timer()
2920 static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_stop_heartbeat_timer()  argument
2922 	del_timer_sync(&ctrl_info->heartbeat_timer);  in pqi_stop_heartbeat_timer()
2941 static unsigned int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)  in pqi_process_event_intr()  argument
2951 	event_queue = &ctrl_info->event_queue;  in pqi_process_event_intr()
2969 				event = &ctrl_info->events[event_index];  in pqi_process_event_intr()
2984 		schedule_work(&ctrl_info->event_work);  in pqi_process_event_intr()
2992 static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_legacy_intx()  argument
2999 	pqi_registers = ctrl_info->pqi_registers;  in pqi_configure_legacy_intx()
3011 static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_change_irq_mode()  argument
3014 	switch (ctrl_info->irq_mode) {  in pqi_change_irq_mode()
3020 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3021 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3030 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3031 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3036 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3043 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3046 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3047 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3055 	ctrl_info->irq_mode = new_mode;  in pqi_change_irq_mode()
3060 static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)  in pqi_is_valid_irq()  argument
3065 	switch (ctrl_info->irq_mode) {  in pqi_is_valid_irq()
3071 			readl(&ctrl_info->pqi_registers->legacy_intx_status);  in pqi_is_valid_irq()
3088 	struct pqi_ctrl_info *ctrl_info;  in pqi_irq_handler()  local
3093 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3095 	if (!pqi_is_valid_irq(ctrl_info))  in pqi_irq_handler()
3098 	num_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
3100 	if (irq == ctrl_info->event_irq)  in pqi_irq_handler()
3101 		num_responses_handled += pqi_process_event_intr(ctrl_info);  in pqi_irq_handler()
3104 		atomic_inc(&ctrl_info->num_interrupts);  in pqi_irq_handler()
3106 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
3107 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
3112 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_request_irqs()  argument
3114 	struct pci_dev *pci_dev = ctrl_info->pci_dev;  in pqi_request_irqs()
3118 	ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);  in pqi_request_irqs()
3120 	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {  in pqi_request_irqs()
3122 			DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);  in pqi_request_irqs()
3129 		ctrl_info->num_msix_vectors_initialized++;  in pqi_request_irqs()
3135 static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_free_irqs()  argument
3139 	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)  in pqi_free_irqs()
3140 		free_irq(pci_irq_vector(ctrl_info->pci_dev, i),  in pqi_free_irqs()
3141 			&ctrl_info->queue_groups[i]);  in pqi_free_irqs()
3143 	ctrl_info->num_msix_vectors_initialized = 0;  in pqi_free_irqs()
3146 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_msix_interrupts()  argument
3150 	num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,  in pqi_enable_msix_interrupts()
3151 			PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,  in pqi_enable_msix_interrupts()
3154 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_enable_msix_interrupts()
3160 	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;  in pqi_enable_msix_interrupts()
3161 	ctrl_info->irq_mode = IRQ_MODE_MSIX;  in pqi_enable_msix_interrupts()
3165 static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_msix_interrupts()  argument
3167 	if (ctrl_info->num_msix_vectors_enabled) {  in pqi_disable_msix_interrupts()
3168 		pci_free_irq_vectors(ctrl_info->pci_dev);  in pqi_disable_msix_interrupts()
3169 		ctrl_info->num_msix_vectors_enabled = 0;  in pqi_disable_msix_interrupts()
3173 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_operational_queues()  argument
3189 		ctrl_info->num_elements_per_iq;  in pqi_alloc_operational_queues()
3192 		ctrl_info->num_elements_per_oq;  in pqi_alloc_operational_queues()
3193 	num_inbound_queues = ctrl_info->num_queue_groups * 2;  in pqi_alloc_operational_queues()
3194 	num_outbound_queues = ctrl_info->num_queue_groups;  in pqi_alloc_operational_queues()
3195 	num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;  in pqi_alloc_operational_queues()
3227 	ctrl_info->queue_memory_base =  in pqi_alloc_operational_queues()
3228 		dma_zalloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_operational_queues()
3230 			&ctrl_info->queue_memory_base_dma_handle, GFP_KERNEL);  in pqi_alloc_operational_queues()
3232 	if (!ctrl_info->queue_memory_base)  in pqi_alloc_operational_queues()
3235 	ctrl_info->queue_memory_length = alloc_length;  in pqi_alloc_operational_queues()
3237 	element_array = PTR_ALIGN(ctrl_info->queue_memory_base,  in pqi_alloc_operational_queues()
3240 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3241 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3244 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3245 				(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3251 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3252 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3258 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3259 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3262 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3263 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3269 	ctrl_info->event_queue.oq_element_array = element_array;  in pqi_alloc_operational_queues()
3270 	ctrl_info->event_queue.oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
3271 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3272 		(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3279 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3280 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3283 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3285 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3291 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3293 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3299 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3301 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3307 	ctrl_info->event_queue.oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
3308 	ctrl_info->event_queue.oq_pi_bus_addr =  in pqi_alloc_operational_queues()
3309 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3311 		(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3316 static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_init_operational_queues()  argument
3326 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3327 		ctrl_info->queue_groups[i].ctrl_info = ctrl_info;  in pqi_init_operational_queues()
3334 	ctrl_info->event_queue.oq_id = next_oq_id++;  in pqi_init_operational_queues()
3335 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3336 		ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3337 		ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3338 		ctrl_info->queue_groups[i].oq_id = next_oq_id++;  in pqi_init_operational_queues()
3345 	ctrl_info->event_queue.int_msg_num = 0;  in pqi_init_operational_queues()
3346 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3347 		ctrl_info->queue_groups[i].int_msg_num = i;  in pqi_init_operational_queues()
3349 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3350 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);  in pqi_init_operational_queues()
3351 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);  in pqi_init_operational_queues()
3352 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);  in pqi_init_operational_queues()
3353 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);  in pqi_init_operational_queues()
3357 static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_admin_queues()  argument
3366 	ctrl_info->admin_queue_memory_base =  in pqi_alloc_admin_queues()
3367 		dma_zalloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_admin_queues()
3369 			&ctrl_info->admin_queue_memory_base_dma_handle,  in pqi_alloc_admin_queues()
3372 	if (!ctrl_info->admin_queue_memory_base)  in pqi_alloc_admin_queues()
3375 	ctrl_info->admin_queue_memory_length = alloc_length;  in pqi_alloc_admin_queues()
3377 	admin_queues = &ctrl_info->admin_queues;  in pqi_alloc_admin_queues()
3378 	admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,  in pqi_alloc_admin_queues()
3389 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3391 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3393 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3395 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3397 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3399 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3401 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3403 		(void __iomem *)ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3411 static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_admin_queues()  argument
3419 	pqi_registers = ctrl_info->pqi_registers;  in pqi_create_admin_queues()
3420 	admin_queues = &ctrl_info->admin_queues;  in pqi_create_admin_queues()
3453 	admin_queues->iq_pi = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3456 	admin_queues->oq_ci = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3463 static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request()  argument
3470 	admin_queues = &ctrl_info->admin_queues;  in pqi_submit_admin_request()
3490 static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,  in pqi_poll_for_admin_response()  argument
3498 	admin_queues = &ctrl_info->admin_queues;  in pqi_poll_for_admin_response()
3508 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_admin_response()
3512 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_admin_response()
3527 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,  in pqi_start_io()  argument
3566 			ctrl_info->num_elements_per_iq))  in pqi_start_io()
3576 			ctrl_info->num_elements_per_iq - iq_pi;  in pqi_start_io()
3590 			ctrl_info->num_elements_per_iq;  in pqi_start_io()
3609 static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_completion_io()  argument
3621 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_completion_io()
3622 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_completion_io()
3662 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_raid_request_synchronous()  argument
3679 		if (down_interruptible(&ctrl_info->sync_request_sem))  in pqi_submit_raid_request_synchronous()
3683 			down(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
3686 			if (down_timeout(&ctrl_info->sync_request_sem,  in pqi_submit_raid_request_synchronous()
3697 	pqi_ctrl_busy(ctrl_info);  in pqi_submit_raid_request_synchronous()
3698 	timeout_msecs = pqi_wait_if_ctrl_blocked(ctrl_info, timeout_msecs);  in pqi_submit_raid_request_synchronous()
3700 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
3705 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_submit_raid_request_synchronous()
3706 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
3711 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_submit_raid_request_synchronous()
3727 	pqi_start_io(ctrl_info,  in pqi_submit_raid_request_synchronous()
3728 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_submit_raid_request_synchronous()
3731 	pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
3734 		pqi_wait_for_completion_io(ctrl_info, &wait);  in pqi_submit_raid_request_synchronous()
3738 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_submit_raid_request_synchronous()
3758 	up(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
3783 	struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request_synchronous()  argument
3789 	pqi_submit_admin_request(ctrl_info, request);  in pqi_submit_admin_request_synchronous()
3791 	rc = pqi_poll_for_admin_response(ctrl_info, response);  in pqi_submit_admin_request_synchronous()
3800 static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_report_device_capability()  argument
3822 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_report_device_capability()
3829 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_report_device_capability()
3832 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_report_device_capability()
3844 	ctrl_info->max_inbound_queues =  in pqi_report_device_capability()
3846 	ctrl_info->max_elements_per_iq =  in pqi_report_device_capability()
3848 	ctrl_info->max_iq_element_length =  in pqi_report_device_capability()
3851 	ctrl_info->max_outbound_queues =  in pqi_report_device_capability()
3853 	ctrl_info->max_elements_per_oq =  in pqi_report_device_capability()
3855 	ctrl_info->max_oq_element_length =  in pqi_report_device_capability()
3862 	ctrl_info->max_inbound_iu_length_per_firmware =  in pqi_report_device_capability()
3865 	ctrl_info->inbound_spanning_supported =  in pqi_report_device_capability()
3867 	ctrl_info->outbound_spanning_supported =  in pqi_report_device_capability()
3876 static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_validate_device_capability()  argument
3878 	if (ctrl_info->max_iq_element_length <  in pqi_validate_device_capability()
3880 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
3882 			ctrl_info->max_iq_element_length,  in pqi_validate_device_capability()
3887 	if (ctrl_info->max_oq_element_length <  in pqi_validate_device_capability()
3889 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
3891 			ctrl_info->max_oq_element_length,  in pqi_validate_device_capability()
3896 	if (ctrl_info->max_inbound_iu_length_per_firmware <  in pqi_validate_device_capability()
3898 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
3900 			ctrl_info->max_inbound_iu_length_per_firmware,  in pqi_validate_device_capability()
3905 	if (!ctrl_info->inbound_spanning_supported) {  in pqi_validate_device_capability()
3906 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
3911 	if (ctrl_info->outbound_spanning_supported) {  in pqi_validate_device_capability()
3912 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
3920 static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)  in pqi_create_event_queue()  argument
3927 	event_queue = &ctrl_info->event_queue;  in pqi_create_event_queue()
3952 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_event_queue()
3957 	event_queue->oq_ci = ctrl_info->iomem_base +  in pqi_create_event_queue()
3965 static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,  in pqi_create_queue_group()  argument
3973 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
3991 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
3997 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4000 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4005 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4026 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4032 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4035 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4040 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4060 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4063 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4082 	put_unaligned_le16(ctrl_info->num_elements_per_oq,  in pqi_create_queue_group()
4090 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4093 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4098 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
4106 static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_queues()  argument
4111 	rc = pqi_create_event_queue(ctrl_info);  in pqi_create_queues()
4113 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4118 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_create_queues()
4119 		rc = pqi_create_queue_group(ctrl_info, i);  in pqi_create_queues()
4121 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4123 				i, ctrl_info->num_queue_groups);  in pqi_create_queues()
4135 static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_events()  argument
4158 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4165 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_configure_events()
4168 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4179 			put_unaligned_le16(ctrl_info->event_queue.oq_id,  in pqi_configure_events()
4194 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4201 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_configure_events()
4204 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4214 static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_events()  argument
4216 	return pqi_configure_events(ctrl_info, true);  in pqi_enable_events()
4219 static inline int pqi_disable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_events()  argument
4221 	return pqi_configure_events(ctrl_info, false);  in pqi_disable_events()
4224 static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_free_all_io_requests()  argument
4231 	if (!ctrl_info->io_request_pool)  in pqi_free_all_io_requests()
4234 	dev = &ctrl_info->pci_dev->dev;  in pqi_free_all_io_requests()
4235 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_free_all_io_requests()
4236 	io_request = ctrl_info->io_request_pool;  in pqi_free_all_io_requests()
4238 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_free_all_io_requests()
4248 	kfree(ctrl_info->io_request_pool);  in pqi_free_all_io_requests()
4249 	ctrl_info->io_request_pool = NULL;  in pqi_free_all_io_requests()
4252 static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_error_buffer()  argument
4254 	ctrl_info->error_buffer = dma_zalloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_error_buffer()
4255 		ctrl_info->error_buffer_length,  in pqi_alloc_error_buffer()
4256 		&ctrl_info->error_buffer_dma_handle, GFP_KERNEL);  in pqi_alloc_error_buffer()
4258 	if (!ctrl_info->error_buffer)  in pqi_alloc_error_buffer()
4264 static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_resources()  argument
4273 	ctrl_info->io_request_pool =  in pqi_alloc_io_resources()
4274 		kcalloc(ctrl_info->max_io_slots,  in pqi_alloc_io_resources()
4275 			sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);  in pqi_alloc_io_resources()
4277 	if (!ctrl_info->io_request_pool) {  in pqi_alloc_io_resources()
4278 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4283 	dev = &ctrl_info->pci_dev->dev;  in pqi_alloc_io_resources()
4284 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_alloc_io_resources()
4285 	io_request = ctrl_info->io_request_pool;  in pqi_alloc_io_resources()
4287 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_alloc_io_resources()
4289 			kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);  in pqi_alloc_io_resources()
4292 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4302 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4317 	pqi_free_all_io_requests(ctrl_info);  in pqi_alloc_io_resources()
4327 static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_io_resources()  argument
4332 	ctrl_info->scsi_ml_can_queue =  in pqi_calculate_io_resources()
4333 		ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;  in pqi_calculate_io_resources()
4334 	ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;  in pqi_calculate_io_resources()
4336 	ctrl_info->error_buffer_length =  in pqi_calculate_io_resources()
4337 		ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;  in pqi_calculate_io_resources()
4340 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4343 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4351 	max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);  in pqi_calculate_io_resources()
4355 	ctrl_info->sg_chain_buffer_length =  in pqi_calculate_io_resources()
4358 	ctrl_info->sg_tablesize = max_sg_entries;  in pqi_calculate_io_resources()
4359 	ctrl_info->max_sectors = max_transfer_size / 512;  in pqi_calculate_io_resources()
4362 static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_queue_resources()  argument
4374 		max_queue_groups = min(ctrl_info->max_inbound_queues / 2,  in pqi_calculate_queue_resources()
4375 			ctrl_info->max_outbound_queues - 1);  in pqi_calculate_queue_resources()
4379 		num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);  in pqi_calculate_queue_resources()
4383 	ctrl_info->num_queue_groups = num_queue_groups;  in pqi_calculate_queue_resources()
4384 	ctrl_info->max_hw_queue_index = num_queue_groups - 1;  in pqi_calculate_queue_resources()
4390 	ctrl_info->max_inbound_iu_length =  in pqi_calculate_queue_resources()
4391 		(ctrl_info->max_inbound_iu_length_per_firmware /  in pqi_calculate_queue_resources()
4396 		(ctrl_info->max_inbound_iu_length /  in pqi_calculate_queue_resources()
4403 		ctrl_info->max_elements_per_iq);  in pqi_calculate_queue_resources()
4407 		ctrl_info->max_elements_per_oq);  in pqi_calculate_queue_resources()
4409 	ctrl_info->num_elements_per_iq = num_elements_per_iq;  in pqi_calculate_queue_resources()
4410 	ctrl_info->num_elements_per_oq = num_elements_per_oq;  in pqi_calculate_queue_resources()
4412 	ctrl_info->max_sg_per_iu =  in pqi_calculate_queue_resources()
4413 		((ctrl_info->max_inbound_iu_length -  in pqi_calculate_queue_resources()
4430 static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_sg_list()  argument
4455 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_raid_sg_list()
4494 static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_sg_list()  argument
4520 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_aio_sg_list()
4571 	struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,  in pqi_raid_submit_scsi_cmd_with_io_request()  argument
4643 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_raid_submit_scsi_cmd_with_io_request()
4649 	rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
4655 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
4660 static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_submit_scsi_cmd()  argument
4666 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_raid_submit_scsi_cmd()
4668 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_raid_submit_scsi_cmd()
4672 static inline void pqi_schedule_bypass_retry(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_bypass_retry()  argument
4674 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_schedule_bypass_retry()
4675 		schedule_work(&ctrl_info->raid_bypass_retry_work);  in pqi_schedule_bypass_retry()
4682 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_needed()  local
4697 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_raid_bypass_retry_needed()
4698 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_bypass_retry_needed()
4705 	struct pqi_ctrl_info *ctrl_info,  in pqi_add_to_raid_bypass_retry_list()  argument
4710 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
4713 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
4716 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
4717 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
4733 	struct pqi_ctrl_info *ctrl_info;  in pqi_queue_raid_bypass_retry()  local
4738 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_queue_raid_bypass_retry()
4740 	pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request, false);  in pqi_queue_raid_bypass_retry()
4741 	pqi_schedule_bypass_retry(ctrl_info);  in pqi_queue_raid_bypass_retry()
4748 	struct pqi_ctrl_info *ctrl_info;  in pqi_retry_raid_bypass()  local
4760 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_retry_raid_bypass()
4765 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_retry_raid_bypass()
4770 	struct pqi_ctrl_info *ctrl_info)  in pqi_next_queued_raid_bypass_request()  argument
4775 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
4777 		&ctrl_info->raid_bypass_retry_list,  in pqi_next_queued_raid_bypass_request()
4781 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
4786 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_retry_raid_bypass_requests()  argument
4791 	pqi_ctrl_busy(ctrl_info);  in pqi_retry_raid_bypass_requests()
4794 		if (pqi_ctrl_blocked(ctrl_info))  in pqi_retry_raid_bypass_requests()
4796 		io_request = pqi_next_queued_raid_bypass_request(ctrl_info);  in pqi_retry_raid_bypass_requests()
4801 			pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request,  in pqi_retry_raid_bypass_requests()
4803 			pqi_schedule_bypass_retry(ctrl_info);  in pqi_retry_raid_bypass_requests()
4808 	pqi_ctrl_unbusy(ctrl_info);  in pqi_retry_raid_bypass_requests()
4813 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_worker()  local
4815 	ctrl_info = container_of(work, struct pqi_ctrl_info,  in pqi_raid_bypass_retry_worker()
4817 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_raid_bypass_retry_worker()
4821 	struct pqi_ctrl_info *ctrl_info)  in pqi_clear_all_queued_raid_bypass_retries()  argument
4825 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
4826 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_clear_all_queued_raid_bypass_retries()
4827 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
4847 static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_scsi_cmd()  argument
4851 	return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,  in pqi_aio_submit_scsi_cmd()
4855 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_io()  argument
4864 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_aio_submit_io()
4898 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_aio_submit_io()
4914 	rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);  in pqi_aio_submit_io()
4920 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
4925 static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,  in pqi_get_hw_queue()  argument
4931 	if (hw_queue > ctrl_info->max_hw_queue_index)  in pqi_get_hw_queue()
4954 	struct pqi_ctrl_info *ctrl_info;  in pqi_scsi_queue_command()  local
4961 	ctrl_info = shost_to_hba(shost);  in pqi_scsi_queue_command()
4965 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_scsi_queue_command()
4971 	pqi_ctrl_busy(ctrl_info);  in pqi_scsi_queue_command()
4972 	if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device)) {  in pqi_scsi_queue_command()
4983 	hw_queue = pqi_get_hw_queue(ctrl_info, scmd);  in pqi_scsi_queue_command()
4984 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
4990 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,  in pqi_scsi_queue_command()
4996 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5000 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5003 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5008 	pqi_ctrl_unbusy(ctrl_info);  in pqi_scsi_queue_command()
5015 static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_until_queued_io_drained()  argument
5032 			pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_queued_io_drained()
5033 			if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_queued_io_drained()
5042 static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_inbound_queues_empty()  argument
5051 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_wait_until_inbound_queues_empty()
5052 		queue_group = &ctrl_info->queue_groups[i];  in pqi_wait_until_inbound_queues_empty()
5054 		rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);  in pqi_wait_until_inbound_queues_empty()
5065 				pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
5066 				if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_inbound_queues_empty()
5076 static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,  in pqi_fail_io_queued_for_device()  argument
5088 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_device()
5089 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
5117 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_device_wait_for_pending_io()  argument
5121 		pqi_check_ctrl_health(ctrl_info);  in pqi_device_wait_for_pending_io()
5122 		if (pqi_ctrl_offline(ctrl_info))  in pqi_device_wait_for_pending_io()
5130 static int pqi_ctrl_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_for_pending_io()  argument
5139 		spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ctrl_wait_for_pending_io()
5140 		list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_ctrl_wait_for_pending_io()
5147 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_ctrl_wait_for_pending_io()
5153 		pqi_check_ctrl_health(ctrl_info);  in pqi_ctrl_wait_for_pending_io()
5154 		if (pqi_ctrl_offline(ctrl_info))  in pqi_ctrl_wait_for_pending_io()
5173 static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_lun_reset_completion()  argument
5185 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_lun_reset_completion()
5186 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_lun_reset_completion()
5195 static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_lun_reset()  argument
5203 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_lun_reset()
5218 	pqi_start_io(ctrl_info,  in pqi_lun_reset()
5219 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_lun_reset()
5222 	rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);  in pqi_lun_reset()
5233 static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_device_reset()  argument
5238 	rc = pqi_lun_reset(ctrl_info, device);  in pqi_device_reset()
5240 		rc = pqi_device_wait_for_pending_io(ctrl_info, device);  in pqi_device_reset()
5249 	struct pqi_ctrl_info *ctrl_info;  in pqi_eh_device_reset_handler()  local
5253 	ctrl_info = shost_to_hba(shost);  in pqi_eh_device_reset_handler()
5256 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5260 	pqi_check_ctrl_health(ctrl_info);  in pqi_eh_device_reset_handler()
5261 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_eh_device_reset_handler()
5266 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_eh_device_reset_handler()
5268 	pqi_ctrl_block_requests(ctrl_info);  in pqi_eh_device_reset_handler()
5269 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_eh_device_reset_handler()
5270 	pqi_fail_io_queued_for_device(ctrl_info, device);  in pqi_eh_device_reset_handler()
5271 	rc = pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_eh_device_reset_handler()
5273 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_eh_device_reset_handler()
5278 		rc = pqi_device_reset(ctrl_info, device);  in pqi_eh_device_reset_handler()
5282 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_eh_device_reset_handler()
5285 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5297 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_alloc()  local
5301 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_alloc()
5303 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5308 		device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);  in pqi_slave_alloc()
5315 		device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),  in pqi_slave_alloc()
5329 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5336 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_map_queues()  local
5338 	return blk_mq_pci_map_queues(&shost->tag_set, ctrl_info->pci_dev, 0);  in pqi_map_queues()
5341 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info,  in pqi_getpciinfo_ioctl()  argument
5352 	pci_dev = ctrl_info->pci_dev;  in pqi_getpciinfo_ioctl()
5453 static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_passthru_ioctl()  argument
5464 	if (pqi_ctrl_offline(ctrl_info))  in pqi_passthru_ioctl()
5535 		rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_passthru_ioctl()
5546 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_passthru_ioctl()
5550 		pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_passthru_ioctl()
5594 	struct pqi_ctrl_info *ctrl_info;  in pqi_ioctl()  local
5596 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ioctl()
5602 		rc = pqi_scan_scsi_devices(ctrl_info);  in pqi_ioctl()
5605 		rc = pqi_getpciinfo_ioctl(ctrl_info, arg);  in pqi_ioctl()
5611 		rc = pqi_passthru_ioctl(ctrl_info, arg);  in pqi_ioctl()
5626 	struct pqi_ctrl_info *ctrl_info;  in pqi_version_show()  local
5629 	ctrl_info = shost_to_hba(shost);  in pqi_version_show()
5635 		"firmware: %s\n", ctrl_info->firmware_version);  in pqi_version_show()
5705 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_address_show()  local
5712 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_address_show()
5714 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
5718 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_sas_address_show()
5724 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
5732 	struct pqi_ctrl_info *ctrl_info;  in pqi_ssd_smart_path_enabled_show()  local
5738 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ssd_smart_path_enabled_show()
5740 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
5747 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
5755 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_level_show()  local
5762 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_level_show()
5764 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
5773 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
5807 static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_register_scsi()  argument
5812 	shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));  in pqi_register_scsi()
5814 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
5816 			ctrl_info->ctrl_id);  in pqi_register_scsi()
5827 	shost->max_sectors = ctrl_info->max_sectors;  in pqi_register_scsi()
5828 	shost->can_queue = ctrl_info->scsi_ml_can_queue;  in pqi_register_scsi()
5830 	shost->sg_tablesize = ctrl_info->sg_tablesize;  in pqi_register_scsi()
5832 	shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);  in pqi_register_scsi()
5834 	shost->nr_hw_queues = ctrl_info->num_queue_groups;  in pqi_register_scsi()
5835 	shost->hostdata[0] = (unsigned long)ctrl_info;  in pqi_register_scsi()
5837 	rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);  in pqi_register_scsi()
5839 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
5841 			ctrl_info->ctrl_id);  in pqi_register_scsi()
5845 	rc = pqi_add_sas_host(shost, ctrl_info);  in pqi_register_scsi()
5847 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
5849 			ctrl_info->ctrl_id);  in pqi_register_scsi()
5853 	ctrl_info->scsi_host = shost;  in pqi_register_scsi()
5865 static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_unregister_scsi()  argument
5869 	pqi_delete_sas_host(ctrl_info);  in pqi_unregister_scsi()
5871 	shost = ctrl_info->scsi_host;  in pqi_unregister_scsi()
5879 static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_reset_completion()  argument
5887 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_reset_completion()
5896 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_pqi_reset_completion()
5897 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_pqi_reset_completion()
5910 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_reset()  argument
5915 	if (ctrl_info->pqi_reset_quiesce_supported) {  in pqi_reset()
5916 		rc = sis_pqi_reset_quiesce(ctrl_info);  in pqi_reset()
5918 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
5929 	writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);  in pqi_reset()
5931 	rc = pqi_wait_for_pqi_reset_completion(ctrl_info);  in pqi_reset()
5933 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
5939 static int pqi_get_ctrl_firmware_version(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_firmware_version()  argument
5948 	rc = pqi_identify_controller(ctrl_info, identify);  in pqi_get_ctrl_firmware_version()
5952 	memcpy(ctrl_info->firmware_version, identify->firmware_version,  in pqi_get_ctrl_firmware_version()
5954 	ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';  in pqi_get_ctrl_firmware_version()
5955 	snprintf(ctrl_info->firmware_version +  in pqi_get_ctrl_firmware_version()
5956 		strlen(ctrl_info->firmware_version),  in pqi_get_ctrl_firmware_version()
5957 		sizeof(ctrl_info->firmware_version),  in pqi_get_ctrl_firmware_version()
5966 static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)  in pqi_process_config_table()  argument
5974 	table_length = ctrl_info->config_table_length;  in pqi_process_config_table()
5978 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
5987 	table_iomem_addr = ctrl_info->iomem_base +  in pqi_process_config_table()
5988 		ctrl_info->config_table_offset;  in pqi_process_config_table()
6000 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
6003 				ctrl_info->heartbeat_counter =  in pqi_process_config_table()
6023 static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_revert_to_sis_mode()  argument
6027 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);  in pqi_revert_to_sis_mode()
6028 	rc = pqi_reset(ctrl_info);  in pqi_revert_to_sis_mode()
6031 	rc = sis_reenable_sis_mode(ctrl_info);  in pqi_revert_to_sis_mode()
6033 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_revert_to_sis_mode()
6037 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_revert_to_sis_mode()
6047 static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_force_sis_mode()  argument
6049 	if (!sis_is_firmware_running(ctrl_info))  in pqi_force_sis_mode()
6052 	if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)  in pqi_force_sis_mode()
6055 	if (sis_is_kernel_up(ctrl_info)) {  in pqi_force_sis_mode()
6056 		pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_force_sis_mode()
6060 	return pqi_revert_to_sis_mode(ctrl_info);  in pqi_force_sis_mode()
6063 static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init()  argument
6067 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init()
6075 	rc = sis_wait_for_ctrl_ready(ctrl_info);  in pqi_ctrl_init()
6083 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init()
6085 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6090 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init()
6092 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6098 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
6100 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
6103 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
6105 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
6109 	pqi_calculate_io_resources(ctrl_info);  in pqi_ctrl_init()
6111 	rc = pqi_alloc_error_buffer(ctrl_info);  in pqi_ctrl_init()
6113 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6123 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init()
6125 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6131 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init()
6133 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6139 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init()
6140 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init()
6142 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init()
6146 	rc = pqi_alloc_admin_queues(ctrl_info);  in pqi_ctrl_init()
6148 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6153 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init()
6155 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6160 	rc = pqi_report_device_capability(ctrl_info);  in pqi_ctrl_init()
6162 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6167 	rc = pqi_validate_device_capability(ctrl_info);  in pqi_ctrl_init()
6171 	pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
6173 	rc = pqi_enable_msix_interrupts(ctrl_info);  in pqi_ctrl_init()
6177 	if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {  in pqi_ctrl_init()
6178 		ctrl_info->max_msix_vectors =  in pqi_ctrl_init()
6179 			ctrl_info->num_msix_vectors_enabled;  in pqi_ctrl_init()
6180 		pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
6183 	rc = pqi_alloc_io_resources(ctrl_info);  in pqi_ctrl_init()
6187 	rc = pqi_alloc_operational_queues(ctrl_info);  in pqi_ctrl_init()
6189 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6194 	pqi_init_operational_queues(ctrl_info);  in pqi_ctrl_init()
6196 	rc = pqi_request_irqs(ctrl_info);  in pqi_ctrl_init()
6200 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init()
6204 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init()
6206 	ctrl_info->controller_online = true;  in pqi_ctrl_init()
6207 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init()
6209 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init()
6211 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6217 	rc = pqi_register_scsi(ctrl_info);  in pqi_ctrl_init()
6221 	rc = pqi_get_ctrl_firmware_version(ctrl_info);  in pqi_ctrl_init()
6223 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6228 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init()
6230 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
6235 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init()
6237 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init()
6242 static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_reinit_queues()  argument
6248 	admin_queues = &ctrl_info->admin_queues;  in pqi_reinit_queues()
6253 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_reinit_queues()
6254 		ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;  in pqi_reinit_queues()
6255 		ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;  in pqi_reinit_queues()
6256 		ctrl_info->queue_groups[i].oq_ci_copy = 0;  in pqi_reinit_queues()
6258 		writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);  in pqi_reinit_queues()
6259 		writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);  in pqi_reinit_queues()
6260 		writel(0, ctrl_info->queue_groups[i].oq_pi);  in pqi_reinit_queues()
6263 	event_queue = &ctrl_info->event_queue;  in pqi_reinit_queues()
6268 static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init_resume()  argument
6272 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init_resume()
6280 	rc = sis_wait_for_ctrl_ready_resume(ctrl_info);  in pqi_ctrl_init_resume()
6289 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init_resume()
6291 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
6297 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init_resume()
6299 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
6305 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init_resume()
6306 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init_resume()
6308 	pqi_reinit_queues(ctrl_info);  in pqi_ctrl_init_resume()
6310 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init_resume()
6312 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
6317 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init_resume()
6321 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init_resume()
6323 	ctrl_info->controller_online = true;  in pqi_ctrl_init_resume()
6324 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init_resume()
6325 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ctrl_init_resume()
6327 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init_resume()
6329 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
6334 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init_resume()
6336 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
6341 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init_resume()
6343 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init_resume()
6355 static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_pci_init()  argument
6360 	rc = pci_enable_device(ctrl_info->pci_dev);  in pqi_pci_init()
6362 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
6372 	rc = dma_set_mask(&ctrl_info->pci_dev->dev, mask);  in pqi_pci_init()
6374 		dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");  in pqi_pci_init()
6378 	rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);  in pqi_pci_init()
6380 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
6385 	ctrl_info->iomem_base = ioremap_nocache(pci_resource_start(  in pqi_pci_init()
6386 		ctrl_info->pci_dev, 0),  in pqi_pci_init()
6388 	if (!ctrl_info->iomem_base) {  in pqi_pci_init()
6389 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
6398 	rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,  in pqi_pci_init()
6401 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
6407 	pci_set_master(ctrl_info->pci_dev);  in pqi_pci_init()
6409 	ctrl_info->registers = ctrl_info->iomem_base;  in pqi_pci_init()
6410 	ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;  in pqi_pci_init()
6412 	pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);  in pqi_pci_init()
6417 	pci_release_regions(ctrl_info->pci_dev);  in pqi_pci_init()
6419 	pci_disable_device(ctrl_info->pci_dev);  in pqi_pci_init()
6424 static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_cleanup_pci_init()  argument
6426 	iounmap(ctrl_info->iomem_base);  in pqi_cleanup_pci_init()
6427 	pci_release_regions(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
6428 	if (pci_is_enabled(ctrl_info->pci_dev))  in pqi_cleanup_pci_init()
6429 		pci_disable_device(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
6430 	pci_set_drvdata(ctrl_info->pci_dev, NULL);  in pqi_cleanup_pci_init()
6435 	struct pqi_ctrl_info *ctrl_info;  in pqi_alloc_ctrl_info()  local
6437 	ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),  in pqi_alloc_ctrl_info()
6439 	if (!ctrl_info)  in pqi_alloc_ctrl_info()
6442 	mutex_init(&ctrl_info->scan_mutex);  in pqi_alloc_ctrl_info()
6443 	mutex_init(&ctrl_info->lun_reset_mutex);  in pqi_alloc_ctrl_info()
6445 	INIT_LIST_HEAD(&ctrl_info->scsi_device_list);  in pqi_alloc_ctrl_info()
6446 	spin_lock_init(&ctrl_info->scsi_device_list_lock);  in pqi_alloc_ctrl_info()
6448 	INIT_WORK(&ctrl_info->event_work, pqi_event_worker);  in pqi_alloc_ctrl_info()
6449 	atomic_set(&ctrl_info->num_interrupts, 0);  in pqi_alloc_ctrl_info()
6451 	INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);  in pqi_alloc_ctrl_info()
6452 	INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);  in pqi_alloc_ctrl_info()
6454 	timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);  in pqi_alloc_ctrl_info()
6455 	INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);  in pqi_alloc_ctrl_info()
6457 	sema_init(&ctrl_info->sync_request_sem,  in pqi_alloc_ctrl_info()
6459 	init_waitqueue_head(&ctrl_info->block_requests_wait);  in pqi_alloc_ctrl_info()
6461 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_alloc_ctrl_info()
6462 	spin_lock_init(&ctrl_info->raid_bypass_retry_list_lock);  in pqi_alloc_ctrl_info()
6463 	INIT_WORK(&ctrl_info->raid_bypass_retry_work,  in pqi_alloc_ctrl_info()
6466 	ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;  in pqi_alloc_ctrl_info()
6467 	ctrl_info->irq_mode = IRQ_MODE_NONE;  in pqi_alloc_ctrl_info()
6468 	ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;  in pqi_alloc_ctrl_info()
6470 	return ctrl_info;  in pqi_alloc_ctrl_info()
6473 static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_info()  argument
6475 	kfree(ctrl_info);  in pqi_free_ctrl_info()
6478 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_free_interrupts()  argument
6480 	pqi_free_irqs(ctrl_info);  in pqi_free_interrupts()
6481 	pqi_disable_msix_interrupts(ctrl_info);  in pqi_free_interrupts()
6484 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_resources()  argument
6486 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_free_ctrl_resources()
6487 	pqi_free_interrupts(ctrl_info);  in pqi_free_ctrl_resources()
6488 	if (ctrl_info->queue_memory_base)  in pqi_free_ctrl_resources()
6489 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
6490 			ctrl_info->queue_memory_length,  in pqi_free_ctrl_resources()
6491 			ctrl_info->queue_memory_base,  in pqi_free_ctrl_resources()
6492 			ctrl_info->queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
6493 	if (ctrl_info->admin_queue_memory_base)  in pqi_free_ctrl_resources()
6494 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
6495 			ctrl_info->admin_queue_memory_length,  in pqi_free_ctrl_resources()
6496 			ctrl_info->admin_queue_memory_base,  in pqi_free_ctrl_resources()
6497 			ctrl_info->admin_queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
6498 	pqi_free_all_io_requests(ctrl_info);  in pqi_free_ctrl_resources()
6499 	if (ctrl_info->error_buffer)  in pqi_free_ctrl_resources()
6500 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
6501 			ctrl_info->error_buffer_length,  in pqi_free_ctrl_resources()
6502 			ctrl_info->error_buffer,  in pqi_free_ctrl_resources()
6503 			ctrl_info->error_buffer_dma_handle);  in pqi_free_ctrl_resources()
6504 	if (ctrl_info->iomem_base)  in pqi_free_ctrl_resources()
6505 		pqi_cleanup_pci_init(ctrl_info);  in pqi_free_ctrl_resources()
6506 	pqi_free_ctrl_info(ctrl_info);  in pqi_free_ctrl_resources()
6509 static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_ctrl()  argument
6511 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_remove_ctrl()
6512 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_remove_ctrl()
6513 	pqi_remove_all_scsi_devices(ctrl_info);  in pqi_remove_ctrl()
6514 	pqi_unregister_scsi(ctrl_info);  in pqi_remove_ctrl()
6515 	if (ctrl_info->pqi_mode_enabled)  in pqi_remove_ctrl()
6516 		pqi_revert_to_sis_mode(ctrl_info);  in pqi_remove_ctrl()
6517 	pqi_free_ctrl_resources(ctrl_info);  in pqi_remove_ctrl()
6540 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_all_outstanding_requests()  argument
6546 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_fail_all_outstanding_requests()
6547 		io_request = &ctrl_info->io_request_pool[i];  in pqi_fail_all_outstanding_requests()
6565 static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline_deferred()  argument
6568 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6569 	pqi_free_interrupts(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6570 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6571 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6572 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6573 	pqi_fail_all_outstanding_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6574 	pqi_clear_all_queued_raid_bypass_retries(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6575 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
6580 	struct pqi_ctrl_info *ctrl_info;  in pqi_ctrl_offline_worker()  local
6582 	ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);  in pqi_ctrl_offline_worker()
6583 	pqi_take_ctrl_offline_deferred(ctrl_info);  in pqi_ctrl_offline_worker()
6586 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline()  argument
6588 	if (!ctrl_info->controller_online)  in pqi_take_ctrl_offline()
6591 	ctrl_info->controller_online = false;  in pqi_take_ctrl_offline()
6592 	ctrl_info->pqi_mode_enabled = false;  in pqi_take_ctrl_offline()
6593 	pqi_ctrl_block_requests(ctrl_info);  in pqi_take_ctrl_offline()
6595 		sis_shutdown_ctrl(ctrl_info);  in pqi_take_ctrl_offline()
6596 	pci_disable_device(ctrl_info->pci_dev);  in pqi_take_ctrl_offline()
6597 	dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");  in pqi_take_ctrl_offline()
6598 	schedule_work(&ctrl_info->ctrl_offline_work);  in pqi_take_ctrl_offline()
6619 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_probe()  local
6639 	ctrl_info = pqi_alloc_ctrl_info(node);  in pqi_pci_probe()
6640 	if (!ctrl_info) {  in pqi_pci_probe()
6646 	ctrl_info->pci_dev = pci_dev;  in pqi_pci_probe()
6648 	rc = pqi_pci_init(ctrl_info);  in pqi_pci_probe()
6652 	rc = pqi_ctrl_init(ctrl_info);  in pqi_pci_probe()
6659 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_probe()
6666 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_remove()  local
6668 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_pci_remove()
6669 	if (!ctrl_info)  in pqi_pci_remove()
6672 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_remove()
6678 	struct pqi_ctrl_info *ctrl_info;  in pqi_shutdown()  local
6680 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_shutdown()
6681 	if (!ctrl_info)  in pqi_shutdown()
6688 	rc = pqi_flush_cache(ctrl_info, SHUTDOWN);  in pqi_shutdown()
6689 	pqi_reset(ctrl_info);  in pqi_shutdown()
6724 	struct pqi_ctrl_info *ctrl_info;  in pqi_suspend()  local
6726 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_suspend()
6728 	pqi_disable_events(ctrl_info);  in pqi_suspend()
6729 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_suspend()
6730 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_suspend()
6731 	pqi_wait_until_scan_finished(ctrl_info);  in pqi_suspend()
6732 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_suspend()
6733 	pqi_flush_cache(ctrl_info, SUSPEND);  in pqi_suspend()
6734 	pqi_ctrl_block_requests(ctrl_info);  in pqi_suspend()
6735 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_suspend()
6736 	pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_suspend()
6737 	pqi_ctrl_wait_for_pending_io(ctrl_info);  in pqi_suspend()
6738 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_suspend()
6746 	ctrl_info->controller_online = false;  in pqi_suspend()
6747 	ctrl_info->pqi_mode_enabled = false;  in pqi_suspend()
6755 	struct pqi_ctrl_info *ctrl_info;  in pqi_resume()  local
6757 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_resume()
6760 		ctrl_info->max_hw_queue_index = 0;  in pqi_resume()
6761 		pqi_free_interrupts(ctrl_info);  in pqi_resume()
6762 		pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);  in pqi_resume()
6765 			&ctrl_info->queue_groups[0]);  in pqi_resume()
6767 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_resume()
6772 		pqi_start_heartbeat_timer(ctrl_info);  in pqi_resume()
6773 		pqi_ctrl_unblock_requests(ctrl_info);  in pqi_resume()
6780 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_resume()