55 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
57 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info);
58 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
60 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
63 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
66 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
70 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
71 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
72 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info);
73 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,
75 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
76 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
77 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
214 static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)  in pqi_check_ctrl_health()  argument
216 	if (ctrl_info->controller_online)  in pqi_check_ctrl_health()
217 		if (!sis_is_firmware_running(ctrl_info))  in pqi_check_ctrl_health()
218 			pqi_take_ctrl_offline(ctrl_info);  in pqi_check_ctrl_health()
227 	struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_mode()  argument
229 	return sis_read_driver_scratch(ctrl_info);  in pqi_get_ctrl_mode()
232 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_save_ctrl_mode()  argument
235 	sis_write_driver_scratch(ctrl_info, mode);  in pqi_save_ctrl_mode()
238 static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_requests()  argument
240 	ctrl_info->block_requests = true;  in pqi_ctrl_block_requests()
241 	scsi_block_requests(ctrl_info->scsi_host);  in pqi_ctrl_block_requests()
244 static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_requests()  argument
246 	ctrl_info->block_requests = false;  in pqi_ctrl_unblock_requests()
247 	wake_up_all(&ctrl_info->block_requests_wait);  in pqi_ctrl_unblock_requests()
248 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_ctrl_unblock_requests()
249 	scsi_unblock_requests(ctrl_info->scsi_host);  in pqi_ctrl_unblock_requests()
252 static unsigned long pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_if_ctrl_blocked()  argument
257 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_wait_if_ctrl_blocked()
260 	atomic_inc(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
263 		wait_event(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
264 			!pqi_ctrl_blocked(ctrl_info));  in pqi_wait_if_ctrl_blocked()
270 			wait_event_timeout(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
271 				!pqi_ctrl_blocked(ctrl_info),  in pqi_wait_if_ctrl_blocked()
276 	atomic_dec(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
281 static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_until_quiesced()  argument
283 	while (atomic_read(&ctrl_info->num_busy_threads) >  in pqi_ctrl_wait_until_quiesced()
284 		atomic_read(&ctrl_info->num_blocked_threads))  in pqi_ctrl_wait_until_quiesced()
308 static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_start()  argument
310 	ctrl_info->in_ofa = true;  in pqi_ctrl_ofa_start()
313 static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_done()  argument
315 	ctrl_info->in_ofa = false;  in pqi_ctrl_ofa_done()
318 static inline bool pqi_ctrl_in_ofa(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_in_ofa()  argument
320 	return ctrl_info->in_ofa;  in pqi_ctrl_in_ofa()
328 static inline bool pqi_device_in_remove(struct pqi_ctrl_info *ctrl_info,  in pqi_device_in_remove()  argument
331 	return device->in_remove && !ctrl_info->in_shutdown;  in pqi_device_in_remove()
335 	struct pqi_ctrl_info *ctrl_info, unsigned long delay)  in pqi_schedule_rescan_worker_with_delay()  argument
337 	if (pqi_ctrl_offline(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
339 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
342 	schedule_delayed_work(&ctrl_info->rescan_work, delay);  in pqi_schedule_rescan_worker_with_delay()
345 static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker()  argument
347 	pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);  in pqi_schedule_rescan_worker()
353 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker_delayed()  argument
355 	pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);  in pqi_schedule_rescan_worker_delayed()
358 static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_rescan_worker()  argument
360 	cancel_delayed_work_sync(&ctrl_info->rescan_work);  in pqi_cancel_rescan_worker()
363 static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)  in pqi_read_heartbeat_counter()  argument
365 	if (!ctrl_info->heartbeat_counter)  in pqi_read_heartbeat_counter()
368 	return readl(ctrl_info->heartbeat_counter);  in pqi_read_heartbeat_counter()
371 static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)  in pqi_read_soft_reset_status()  argument
373 	if (!ctrl_info->soft_reset_status)  in pqi_read_soft_reset_status()
376 	return readb(ctrl_info->soft_reset_status);  in pqi_read_soft_reset_status()
379 static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info,  in pqi_clear_soft_reset_status()  argument
384 	if (!ctrl_info->soft_reset_status)  in pqi_clear_soft_reset_status()
387 	status = pqi_read_soft_reset_status(ctrl_info);  in pqi_clear_soft_reset_status()
389 	writeb(status, ctrl_info->soft_reset_status);  in pqi_clear_soft_reset_status()
429 static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_path_request()  argument
510 		dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n",  in pqi_build_raid_path_request()
530 	return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],  in pqi_build_raid_path_request()
543 	struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_request()  argument
546 	u16 i = ctrl_info->next_io_request_slot;	/* benignly racy */  in pqi_alloc_io_request()
549 		io_request = &ctrl_info->io_request_pool[i];  in pqi_alloc_io_request()
553 		i = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
557 	ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;  in pqi_alloc_io_request()
569 static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_send_scsi_raid_request()  argument
578 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_send_scsi_raid_request()
584 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_send_scsi_raid_request()
587 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_send_scsi_raid_request()
593 static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_request()  argument
596 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_request()
600 static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_with_error()  argument
604 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_with_error()
609 static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_controller()  argument
612 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,  in pqi_identify_controller()
616 static inline int pqi_sense_subsystem_info(struct  pqi_ctrl_info *ctrl_info,  in pqi_sense_subsystem_info()  argument
619 	return pqi_send_ctrl_raid_request(ctrl_info,  in pqi_sense_subsystem_info()
624 static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_inquiry()  argument
627 	return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,  in pqi_scsi_inquiry()
631 static bool pqi_vpd_page_supported(struct pqi_ctrl_info *ctrl_info,  in pqi_vpd_page_supported()  argument
644 	rc = pqi_scsi_inquiry(ctrl_info, scsi3addr,  in pqi_vpd_page_supported()
657 	rc = pqi_scsi_inquiry(ctrl_info, scsi3addr,  in pqi_vpd_page_supported()
677 static int pqi_get_device_id(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_id()  argument
683 	if (!pqi_vpd_page_supported(ctrl_info, scsi3addr, SCSI_VPD_DEVICE_ID))  in pqi_get_device_id()
690 	rc = pqi_scsi_inquiry(ctrl_info, scsi3addr,  in pqi_get_device_id()
704 static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_physical_device()  argument
714 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_identify_physical_device()
724 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_identify_physical_device()
727 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_identify_physical_device()
731 static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,  in pqi_flush_cache()  argument
741 	if (pqi_ctrl_offline(ctrl_info))  in pqi_flush_cache()
750 	rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,  in pqi_flush_cache()
758 int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,  in pqi_csmi_smp_passthru()  argument
762 	return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,  in pqi_csmi_smp_passthru()
768 static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)  in pqi_set_diag_rescan()  argument
777 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,  in pqi_set_diag_rescan()
784 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS,  in pqi_set_diag_rescan()
792 static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,  in pqi_write_host_wellness()  argument
795 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,  in pqi_write_host_wellness()
813 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_driver_version_to_host_wellness()  argument
841 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_driver_version_to_host_wellness()
862 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_current_time_to_host_wellness()  argument
904 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_current_time_to_host_wellness()
916 	struct pqi_ctrl_info *ctrl_info;  in pqi_update_time_worker()  local
918 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_update_time_worker()
921 	if (pqi_ctrl_offline(ctrl_info))  in pqi_update_time_worker()
924 	rc = pqi_write_current_time_to_host_wellness(ctrl_info);  in pqi_update_time_worker()
926 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_time_worker()
929 	schedule_delayed_work(&ctrl_info->update_time_work,  in pqi_update_time_worker()
934 	struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_update_time_worker()  argument
936 	schedule_delayed_work(&ctrl_info->update_time_work, 0);  in pqi_schedule_update_time_worker()
940 	struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_update_time_worker()  argument
942 	cancel_delayed_work_sync(&ctrl_info->update_time_work);  in pqi_cancel_update_time_worker()
945 static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_luns()  argument
948 	return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer,  in pqi_report_luns()
952 static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_report_phys_logical_luns()  argument
968 	rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,  in pqi_report_phys_logical_luns()
989 	rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);  in pqi_report_phys_logical_luns()
1015 static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_phys_luns()  argument
1018 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,  in pqi_report_phys_luns()
1022 static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,  in pqi_report_logical_luns()  argument
1025 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);  in pqi_report_logical_luns()
1028 static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_lists()  argument
1039 	rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);  in pqi_get_device_lists()
1041 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1044 	rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);  in pqi_get_device_lists()
1046 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1137 static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_level()  argument
1148 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_level()
1161 static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_validate_raid_map()  argument
1201 	dev_warn(&ctrl_info->pci_dev->dev,  in pqi_validate_raid_map()
1209 static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_map()  argument
1220 	rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1237 		rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1245 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_get_raid_map()
1253 	rc = pqi_validate_raid_map(ctrl_info, device, raid_map);  in pqi_get_raid_map()
1267 static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_bypass_status()  argument
1278 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_bypass_status()
1292 		pqi_get_raid_map(ctrl_info, device) == 0)  in pqi_get_raid_bypass_status()
1303 static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_volume_status()  argument
1317 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_volume_status()
1343 static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_info()  argument
1359 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0,  in pqi_get_device_info()
1382 			pqi_get_raid_level(ctrl_info, device);  in pqi_get_device_info()
1383 			pqi_get_raid_bypass_status(ctrl_info, device);  in pqi_get_device_info()
1384 			pqi_get_volume_status(ctrl_info, device);  in pqi_get_device_info()
1388 	if (pqi_get_device_id(ctrl_info, device->scsi3addr,  in pqi_get_device_info()
1390 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_get_device_info()
1392 			ctrl_info->scsi_host->host_no,  in pqi_get_device_info()
1402 static void pqi_get_physical_disk_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_physical_disk_info()  argument
1410 	rc = pqi_identify_physical_device(ctrl_info, device,  in pqi_get_physical_disk_info()
1433 static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_show_volume_status()  argument
1530 	dev_info(&ctrl_info->pci_dev->dev,  in pqi_show_volume_status()
1532 		ctrl_info->scsi_host->host_no,  in pqi_show_volume_status()
1538 	struct pqi_ctrl_info *ctrl_info;  in pqi_rescan_worker()  local
1540 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_rescan_worker()
1543 	pqi_scan_scsi_devices(ctrl_info);  in pqi_rescan_worker()
1546 static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,  in pqi_add_device()  argument
1552 		rc = scsi_add_device(ctrl_info->scsi_host, device->bus,  in pqi_add_device()
1555 		rc = pqi_add_sas_device(ctrl_info->sas_host, device);  in pqi_add_device()
1562 static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,  in pqi_remove_device()  argument
1569 	rc = pqi_device_wait_for_pending_io(ctrl_info, device,  in pqi_remove_device()
1572 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_remove_device()
1574 			ctrl_info->scsi_host->host_no, device->bus,  in pqi_remove_device()
1586 static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,  in pqi_find_scsi_dev()  argument
1591 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_find_scsi_dev()
1619 static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_find_entry()  argument
1625 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_scsi_find_entry()
1652 static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,  in pqi_dev_info()  argument
1659 		"%d:%d:", ctrl_info->scsi_host->host_no, device->bus);  in pqi_dev_info()
1708 	dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);  in pqi_dev_info()
1779 static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,  in pqi_fixup_botched_add()  argument
1784 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1786 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
1800 static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,  in pqi_update_device_list()  argument
1820 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1823 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1830 		find_result = pqi_scsi_find_entry(ctrl_info, device,  in pqi_update_device_list()
1861 	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1877 			&ctrl_info->scsi_device_list);  in pqi_update_device_list()
1883 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
1885 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_update_device_list()
1886 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_update_device_list()
1892 			pqi_dev_info(ctrl_info, "offline", device);  in pqi_update_device_list()
1893 			pqi_show_volume_status(ctrl_info, device);  in pqi_update_device_list()
1895 			pqi_dev_info(ctrl_info, "removed", device);  in pqi_update_device_list()
1898 			pqi_remove_device(ctrl_info, device);  in pqi_update_device_list()
1907 	list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
1920 			pqi_dev_info(ctrl_info, "added", device);  in pqi_update_device_list()
1921 			rc = pqi_add_device(ctrl_info, device);  in pqi_update_device_list()
1923 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_device_list()
1925 					ctrl_info->scsi_host->host_no,  in pqi_update_device_list()
1928 				pqi_fixup_botched_add(ctrl_info, device);  in pqi_update_device_list()
2004 static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_update_scsi_devices()  argument
2028 	rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);  in pqi_update_scsi_devices()
2055 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2083 		dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);  in pqi_update_scsi_devices()
2091 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2143 		rc = pqi_get_device_info(ctrl_info, device);  in pqi_update_scsi_devices()
2145 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2151 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2156 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2179 				pqi_get_physical_disk_info(ctrl_info,  in pqi_update_scsi_devices()
2193 	pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);  in pqi_update_scsi_devices()
2212 static void pqi_remove_all_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_all_scsi_devices()  argument
2218 		spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_remove_all_scsi_devices()
2220 		device = list_first_entry_or_null(&ctrl_info->scsi_device_list,  in pqi_remove_all_scsi_devices()
2225 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_remove_all_scsi_devices()
2232 			pqi_remove_device(ctrl_info, device);  in pqi_remove_all_scsi_devices()
2237 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_scan_scsi_devices()  argument
2241 	if (pqi_ctrl_offline(ctrl_info))  in pqi_scan_scsi_devices()
2244 	if (!mutex_trylock(&ctrl_info->scan_mutex)) {  in pqi_scan_scsi_devices()
2245 		pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2248 		rc = pqi_update_scsi_devices(ctrl_info);  in pqi_scan_scsi_devices()
2250 			pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2251 		mutex_unlock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
2259 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_start()  local
2261 	ctrl_info = shost_to_hba(shost);  in pqi_scan_start()
2262 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_scan_start()
2265 	pqi_scan_scsi_devices(ctrl_info);  in pqi_scan_start()
2273 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_finished()  local
2275 	ctrl_info = shost_priv(shost);  in pqi_scan_finished()
2277 	return !mutex_is_locked(&ctrl_info->scan_mutex);  in pqi_scan_finished()
2280 static void pqi_wait_until_scan_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_scan_finished()  argument
2282 	mutex_lock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2283 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_wait_until_scan_finished()
2286 static void pqi_wait_until_lun_reset_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_lun_reset_finished()  argument
2288 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2289 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_wait_until_lun_reset_finished()
2292 static void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_ofa_finished()  argument
2294 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
2295 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
2325 static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_bypass_submit_scsi_cmd()  argument
2648 	return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,  in pqi_raid_bypass_submit_scsi_cmd()
2666 static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_mode_ready()  argument
2673 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_mode_ready()
2682 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2694 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2706 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
2727 	struct pqi_ctrl_info *ctrl_info;  in pqi_take_device_offline()  local
2735 	ctrl_info = shost_to_hba(sdev->host);  in pqi_take_device_offline()
2736 	pqi_schedule_rescan_worker(ctrl_info);  in pqi_take_device_offline()
2737 	dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",  in pqi_take_device_offline()
2738 		path, ctrl_info->scsi_host->host_no, device->bus,  in pqi_take_device_offline()
2813 			struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);  in pqi_process_raid_io_error()  local
2820 						ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
2828 						sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
2967 static unsigned int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info,  in pqi_process_io_intr()  argument
2990 		WARN_ON(request_id >= ctrl_info->max_io_slots);  in pqi_process_io_intr()
2992 		io_request = &ctrl_info->io_request_pool[request_id];  in pqi_process_io_intr()
3020 			io_request->error_info = ctrl_info->error_buffer +  in pqi_process_io_intr()
3027 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
3041 		oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;  in pqi_process_io_intr()
3065 static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,  in pqi_send_event_ack()  argument
3074 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
3084 			ctrl_info->num_elements_per_iq))  in pqi_send_event_ack()
3090 		if (pqi_ctrl_offline(ctrl_info))  in pqi_send_event_ack()
3099 	iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;  in pqi_send_event_ack()
3111 static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,  in pqi_acknowledge_event()  argument
3125 	pqi_send_event_ack(ctrl_info, &request, sizeof(request));  in pqi_acknowledge_event()
3132 	struct pqi_ctrl_info *ctrl_info)  in pqi_poll_for_soft_reset_status()  argument
3140 		status = pqi_read_soft_reset_status(ctrl_info);  in pqi_poll_for_soft_reset_status()
3148 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_soft_reset_status()
3153 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_soft_reset_status()
3160 static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_process_soft_reset()  argument
3169 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3170 			"resetting controller %u\n", ctrl_info->ctrl_id);  in pqi_process_soft_reset()
3171 		sis_soft_reset(ctrl_info);  in pqi_process_soft_reset()
3174 		rc = pqi_ofa_ctrl_restart(ctrl_info);  in pqi_process_soft_reset()
3175 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3176 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3178 			ctrl_info->ctrl_id, rc == 0 ? "SUCCESS" : "FAILED");  in pqi_process_soft_reset()
3181 		pqi_ofa_ctrl_unquiesce(ctrl_info);  in pqi_process_soft_reset()
3182 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3184 			ctrl_info->ctrl_id, "ABORTED");  in pqi_process_soft_reset()
3187 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3188 		pqi_take_ctrl_offline(ctrl_info);  in pqi_process_soft_reset()
3193 static void pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_process_event()  argument
3201 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_ofa_process_event()
3204 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3206 			 ctrl_info->ctrl_id);  in pqi_ofa_process_event()
3207 		pqi_ofa_ctrl_quiesce(ctrl_info);  in pqi_ofa_process_event()
3208 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3209 		if (ctrl_info->soft_reset_handshake_supported) {  in pqi_ofa_process_event()
3210 			status = pqi_poll_for_soft_reset_status(ctrl_info);  in pqi_ofa_process_event()
3211 			pqi_process_soft_reset(ctrl_info, status);  in pqi_ofa_process_event()
3213 			pqi_process_soft_reset(ctrl_info,  in pqi_ofa_process_event()
3218 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3219 		pqi_ofa_setup_host_buffer(ctrl_info,  in pqi_ofa_process_event()
3221 		pqi_ofa_host_memory_update(ctrl_info);  in pqi_ofa_process_event()
3223 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_ofa_process_event()
3224 		pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_process_event()
3225 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3227 			 ctrl_info->ctrl_id, event->ofa_cancel_reason);  in pqi_ofa_process_event()
3230 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_ofa_process_event()
3236 	struct pqi_ctrl_info *ctrl_info;  in pqi_event_worker()  local
3239 	ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);  in pqi_event_worker()
3241 	pqi_ctrl_busy(ctrl_info);  in pqi_event_worker()
3242 	pqi_wait_if_ctrl_blocked(ctrl_info, NO_TIMEOUT);  in pqi_event_worker()
3243 	if (pqi_ctrl_offline(ctrl_info))  in pqi_event_worker()
3246 	pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_event_worker()
3248 	event = ctrl_info->events;  in pqi_event_worker()
3253 				pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
3254 				pqi_ofa_process_event(ctrl_info, event);  in pqi_event_worker()
3257 			pqi_acknowledge_event(ctrl_info, event);  in pqi_event_worker()
3263 	pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
3272 	struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t,  in pqi_heartbeat_timer_handler()  local
3275 	pqi_check_ctrl_health(ctrl_info);  in pqi_heartbeat_timer_handler()
3276 	if (pqi_ctrl_offline(ctrl_info))  in pqi_heartbeat_timer_handler()
3279 	num_interrupts = atomic_read(&ctrl_info->num_interrupts);  in pqi_heartbeat_timer_handler()
3280 	heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);  in pqi_heartbeat_timer_handler()
3282 	if (num_interrupts == ctrl_info->previous_num_interrupts) {  in pqi_heartbeat_timer_handler()
3283 		if (heartbeat_count == ctrl_info->previous_heartbeat_count) {  in pqi_heartbeat_timer_handler()
3284 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_heartbeat_timer_handler()
3287 			pqi_take_ctrl_offline(ctrl_info);  in pqi_heartbeat_timer_handler()
3291 		ctrl_info->previous_num_interrupts = num_interrupts;  in pqi_heartbeat_timer_handler()
3294 	ctrl_info->previous_heartbeat_count = heartbeat_count;  in pqi_heartbeat_timer_handler()
3295 	mod_timer(&ctrl_info->heartbeat_timer,  in pqi_heartbeat_timer_handler()
3299 static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_start_heartbeat_timer()  argument
3301 	if (!ctrl_info->heartbeat_counter)  in pqi_start_heartbeat_timer()
3304 	ctrl_info->previous_num_interrupts =  in pqi_start_heartbeat_timer()
3305 		atomic_read(&ctrl_info->num_interrupts);  in pqi_start_heartbeat_timer()
3306 	ctrl_info->previous_heartbeat_count =  in pqi_start_heartbeat_timer()
3307 		pqi_read_heartbeat_counter(ctrl_info);  in pqi_start_heartbeat_timer()
3309 	ctrl_info->heartbeat_timer.expires =  in pqi_start_heartbeat_timer()
3311 	add_timer(&ctrl_info->heartbeat_timer);  in pqi_start_heartbeat_timer()
3314 static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_stop_heartbeat_timer()  argument
3316 	del_timer_sync(&ctrl_info->heartbeat_timer);  in pqi_stop_heartbeat_timer()
3353 static unsigned int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)  in pqi_process_event_intr()  argument
3363 	event_queue = &ctrl_info->event_queue;  in pqi_process_event_intr()
3381 				event = &ctrl_info->events[event_index];  in pqi_process_event_intr()
3397 		schedule_work(&ctrl_info->event_work);  in pqi_process_event_intr()
3405 static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_legacy_intx()  argument
3412 	pqi_registers = ctrl_info->pqi_registers;  in pqi_configure_legacy_intx()
3424 static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_change_irq_mode()  argument
3427 	switch (ctrl_info->irq_mode) {  in pqi_change_irq_mode()
3433 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3434 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3443 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3444 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3449 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3456 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3459 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3460 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3468 	ctrl_info->irq_mode = new_mode;  in pqi_change_irq_mode()
3473 static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)  in pqi_is_valid_irq()  argument
3478 	switch (ctrl_info->irq_mode) {  in pqi_is_valid_irq()
3484 			readl(&ctrl_info->pqi_registers->legacy_intx_status);  in pqi_is_valid_irq()
3501 	struct pqi_ctrl_info *ctrl_info;  in pqi_irq_handler()  local
3506 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3508 	if (!pqi_is_valid_irq(ctrl_info))  in pqi_irq_handler()
3511 	num_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
3513 	if (irq == ctrl_info->event_irq)  in pqi_irq_handler()
3514 		num_responses_handled += pqi_process_event_intr(ctrl_info);  in pqi_irq_handler()
3517 		atomic_inc(&ctrl_info->num_interrupts);  in pqi_irq_handler()
3519 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
3520 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
3525 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_request_irqs()  argument
3527 	struct pci_dev *pci_dev = ctrl_info->pci_dev;  in pqi_request_irqs()
3531 	ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);  in pqi_request_irqs()
3533 	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {  in pqi_request_irqs()
3535 			DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);  in pqi_request_irqs()
3542 		ctrl_info->num_msix_vectors_initialized++;  in pqi_request_irqs()
3548 static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_free_irqs()  argument
3552 	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)  in pqi_free_irqs()
3553 		free_irq(pci_irq_vector(ctrl_info->pci_dev, i),  in pqi_free_irqs()
3554 			&ctrl_info->queue_groups[i]);  in pqi_free_irqs()
3556 	ctrl_info->num_msix_vectors_initialized = 0;  in pqi_free_irqs()
3559 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_msix_interrupts()  argument
3563 	num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,  in pqi_enable_msix_interrupts()
3564 			PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,  in pqi_enable_msix_interrupts()
3567 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_enable_msix_interrupts()
3573 	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;  in pqi_enable_msix_interrupts()
3574 	ctrl_info->irq_mode = IRQ_MODE_MSIX;  in pqi_enable_msix_interrupts()
3578 static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_msix_interrupts()  argument
3580 	if (ctrl_info->num_msix_vectors_enabled) {  in pqi_disable_msix_interrupts()
3581 		pci_free_irq_vectors(ctrl_info->pci_dev);  in pqi_disable_msix_interrupts()
3582 		ctrl_info->num_msix_vectors_enabled = 0;  in pqi_disable_msix_interrupts()
3586 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_operational_queues()  argument
3602 		ctrl_info->num_elements_per_iq;  in pqi_alloc_operational_queues()
3605 		ctrl_info->num_elements_per_oq;  in pqi_alloc_operational_queues()
3606 	num_inbound_queues = ctrl_info->num_queue_groups * 2;  in pqi_alloc_operational_queues()
3607 	num_outbound_queues = ctrl_info->num_queue_groups;  in pqi_alloc_operational_queues()
3608 	num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;  in pqi_alloc_operational_queues()
3640 	ctrl_info->queue_memory_base =  in pqi_alloc_operational_queues()
3641 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_operational_queues()
3642 				   &ctrl_info->queue_memory_base_dma_handle,  in pqi_alloc_operational_queues()
3645 	if (!ctrl_info->queue_memory_base)  in pqi_alloc_operational_queues()
3648 	ctrl_info->queue_memory_length = alloc_length;  in pqi_alloc_operational_queues()
3650 	element_array = PTR_ALIGN(ctrl_info->queue_memory_base,  in pqi_alloc_operational_queues()
3653 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3654 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3657 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3658 				(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3664 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3665 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3671 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3672 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3675 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3676 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3682 	ctrl_info->event_queue.oq_element_array = element_array;  in pqi_alloc_operational_queues()
3683 	ctrl_info->event_queue.oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
3684 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3685 		(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3692 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
3693 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
3696 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3698 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3704 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3706 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3712 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3714 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3720 	ctrl_info->event_queue.oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
3721 	ctrl_info->event_queue.oq_pi_bus_addr =  in pqi_alloc_operational_queues()
3722 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
3724 		(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
3729 static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_init_operational_queues()  argument
3739 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3740 		ctrl_info->queue_groups[i].ctrl_info = ctrl_info;  in pqi_init_operational_queues()
3747 	ctrl_info->event_queue.oq_id = next_oq_id++;  in pqi_init_operational_queues()
3748 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3749 		ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3750 		ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;  in pqi_init_operational_queues()
3751 		ctrl_info->queue_groups[i].oq_id = next_oq_id++;  in pqi_init_operational_queues()
3758 	ctrl_info->event_queue.int_msg_num = 0;  in pqi_init_operational_queues()
3759 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
3760 		ctrl_info->queue_groups[i].int_msg_num = i;  in pqi_init_operational_queues()
3762 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
3763 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);  in pqi_init_operational_queues()
3764 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);  in pqi_init_operational_queues()
3765 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);  in pqi_init_operational_queues()
3766 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);  in pqi_init_operational_queues()
3770 static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_admin_queues()  argument
3779 	ctrl_info->admin_queue_memory_base =  in pqi_alloc_admin_queues()
3780 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_admin_queues()
3781 				   &ctrl_info->admin_queue_memory_base_dma_handle,  in pqi_alloc_admin_queues()
3784 	if (!ctrl_info->admin_queue_memory_base)  in pqi_alloc_admin_queues()
3787 	ctrl_info->admin_queue_memory_length = alloc_length;  in pqi_alloc_admin_queues()
3789 	admin_queues = &ctrl_info->admin_queues;  in pqi_alloc_admin_queues()
3790 	admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,  in pqi_alloc_admin_queues()
3801 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3803 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3805 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3807 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3809 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3811 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3813 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
3815 		(void __iomem *)ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
3823 static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_admin_queues()  argument
3831 	pqi_registers = ctrl_info->pqi_registers;  in pqi_create_admin_queues()
3832 	admin_queues = &ctrl_info->admin_queues;  in pqi_create_admin_queues()
3865 	admin_queues->iq_pi = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3868 	admin_queues->oq_ci = ctrl_info->iomem_base +  in pqi_create_admin_queues()
3875 static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request()  argument
3882 	admin_queues = &ctrl_info->admin_queues;  in pqi_submit_admin_request()
3902 static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,  in pqi_poll_for_admin_response()  argument
3910 	admin_queues = &ctrl_info->admin_queues;  in pqi_poll_for_admin_response()
3920 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_admin_response()
3924 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_admin_response()
3939 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,  in pqi_start_io()  argument
3978 			ctrl_info->num_elements_per_iq))  in pqi_start_io()
3988 			ctrl_info->num_elements_per_iq - iq_pi;  in pqi_start_io()
4002 			ctrl_info->num_elements_per_iq;  in pqi_start_io()
4021 static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_completion_io()  argument
4033 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_completion_io()
4034 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_completion_io()
4074 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_raid_request_synchronous()  argument
4091 		if (down_interruptible(&ctrl_info->sync_request_sem))  in pqi_submit_raid_request_synchronous()
4095 			down(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4098 			if (down_timeout(&ctrl_info->sync_request_sem,  in pqi_submit_raid_request_synchronous()
4111 	pqi_ctrl_busy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4112 	timeout_msecs = pqi_wait_if_ctrl_blocked(ctrl_info, timeout_msecs);  in pqi_submit_raid_request_synchronous()
4114 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4119 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_submit_raid_request_synchronous()
4120 		pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4125 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_submit_raid_request_synchronous()
4141 	pqi_start_io(ctrl_info,  in pqi_submit_raid_request_synchronous()
4142 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_submit_raid_request_synchronous()
4145 	pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4148 		pqi_wait_for_completion_io(ctrl_info, &wait);  in pqi_submit_raid_request_synchronous()
4152 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_submit_raid_request_synchronous()
4172 	up(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4197 	struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request_synchronous()  argument
4203 	pqi_submit_admin_request(ctrl_info, request);  in pqi_submit_admin_request_synchronous()
4205 	rc = pqi_poll_for_admin_response(ctrl_info, response);  in pqi_submit_admin_request_synchronous()
4214 static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_report_device_capability()  argument
4236 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_report_device_capability()
4243 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_report_device_capability()
4246 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_report_device_capability()
4258 	ctrl_info->max_inbound_queues =  in pqi_report_device_capability()
4260 	ctrl_info->max_elements_per_iq =  in pqi_report_device_capability()
4262 	ctrl_info->max_iq_element_length =  in pqi_report_device_capability()
4265 	ctrl_info->max_outbound_queues =  in pqi_report_device_capability()
4267 	ctrl_info->max_elements_per_oq =  in pqi_report_device_capability()
4269 	ctrl_info->max_oq_element_length =  in pqi_report_device_capability()
4276 	ctrl_info->max_inbound_iu_length_per_firmware =  in pqi_report_device_capability()
4279 	ctrl_info->inbound_spanning_supported =  in pqi_report_device_capability()
4281 	ctrl_info->outbound_spanning_supported =  in pqi_report_device_capability()
4290 static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_validate_device_capability()  argument
4292 	if (ctrl_info->max_iq_element_length <  in pqi_validate_device_capability()
4294 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4296 			ctrl_info->max_iq_element_length,  in pqi_validate_device_capability()
4301 	if (ctrl_info->max_oq_element_length <  in pqi_validate_device_capability()
4303 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4305 			ctrl_info->max_oq_element_length,  in pqi_validate_device_capability()
4310 	if (ctrl_info->max_inbound_iu_length_per_firmware <  in pqi_validate_device_capability()
4312 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4314 			ctrl_info->max_inbound_iu_length_per_firmware,  in pqi_validate_device_capability()
4319 	if (!ctrl_info->inbound_spanning_supported) {  in pqi_validate_device_capability()
4320 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4325 	if (ctrl_info->outbound_spanning_supported) {  in pqi_validate_device_capability()
4326 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4334 static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)  in pqi_create_event_queue()  argument
4341 	event_queue = &ctrl_info->event_queue;  in pqi_create_event_queue()
4366 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_event_queue()
4371 	event_queue->oq_ci = ctrl_info->iomem_base +  in pqi_create_event_queue()
4379 static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,  in pqi_create_queue_group()  argument
4387 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
4405 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4411 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4414 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4419 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4440 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4446 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4449 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4454 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4474 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4477 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4496 	put_unaligned_le16(ctrl_info->num_elements_per_oq,  in pqi_create_queue_group()
4504 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4507 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4512 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
4520 static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_queues()  argument
4525 	rc = pqi_create_event_queue(ctrl_info);  in pqi_create_queues()
4527 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4532 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_create_queues()
4533 		rc = pqi_create_queue_group(ctrl_info, i);  in pqi_create_queues()
4535 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4537 				i, ctrl_info->num_queue_groups);  in pqi_create_queues()
4549 static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_events()  argument
4572 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4579 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_configure_events()
4582 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4593 			put_unaligned_le16(ctrl_info->event_queue.oq_id,  in pqi_configure_events()
4608 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
4615 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,  in pqi_configure_events()
4618 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
4628 static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_events()  argument
4630 	return pqi_configure_events(ctrl_info, true);  in pqi_enable_events()
4633 static inline int pqi_disable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_events()  argument
4635 	return pqi_configure_events(ctrl_info, false);  in pqi_disable_events()
4638 static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_free_all_io_requests()  argument
4645 	if (!ctrl_info->io_request_pool)  in pqi_free_all_io_requests()
4648 	dev = &ctrl_info->pci_dev->dev;  in pqi_free_all_io_requests()
4649 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_free_all_io_requests()
4650 	io_request = ctrl_info->io_request_pool;  in pqi_free_all_io_requests()
4652 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_free_all_io_requests()
4662 	kfree(ctrl_info->io_request_pool);  in pqi_free_all_io_requests()
4663 	ctrl_info->io_request_pool = NULL;  in pqi_free_all_io_requests()
4666 static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_error_buffer()  argument
4668 	ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_error_buffer()
4669 						     ctrl_info->error_buffer_length,  in pqi_alloc_error_buffer()
4670 						     &ctrl_info->error_buffer_dma_handle,  in pqi_alloc_error_buffer()
4673 	if (!ctrl_info->error_buffer)  in pqi_alloc_error_buffer()
4679 static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_resources()  argument
4688 	ctrl_info->io_request_pool =  in pqi_alloc_io_resources()
4689 		kcalloc(ctrl_info->max_io_slots,  in pqi_alloc_io_resources()
4690 			sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);  in pqi_alloc_io_resources()
4692 	if (!ctrl_info->io_request_pool) {  in pqi_alloc_io_resources()
4693 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4698 	dev = &ctrl_info->pci_dev->dev;  in pqi_alloc_io_resources()
4699 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_alloc_io_resources()
4700 	io_request = ctrl_info->io_request_pool;  in pqi_alloc_io_resources()
4702 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_alloc_io_resources()
4704 			kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);  in pqi_alloc_io_resources()
4707 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4717 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
4732 	pqi_free_all_io_requests(ctrl_info);  in pqi_alloc_io_resources()
4742 static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_io_resources()  argument
4747 	ctrl_info->scsi_ml_can_queue =  in pqi_calculate_io_resources()
4748 		ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;  in pqi_calculate_io_resources()
4749 	ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;  in pqi_calculate_io_resources()
4751 	ctrl_info->error_buffer_length =  in pqi_calculate_io_resources()
4752 		ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;  in pqi_calculate_io_resources()
4755 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4758 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
4766 	max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);  in pqi_calculate_io_resources()
4770 	ctrl_info->sg_chain_buffer_length =  in pqi_calculate_io_resources()
4773 	ctrl_info->sg_tablesize = max_sg_entries;  in pqi_calculate_io_resources()
4774 	ctrl_info->max_sectors = max_transfer_size / 512;  in pqi_calculate_io_resources()
4777 static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_queue_resources()  argument
4789 		max_queue_groups = min(ctrl_info->max_inbound_queues / 2,  in pqi_calculate_queue_resources()
4790 			ctrl_info->max_outbound_queues - 1);  in pqi_calculate_queue_resources()
4794 		num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);  in pqi_calculate_queue_resources()
4798 	ctrl_info->num_queue_groups = num_queue_groups;  in pqi_calculate_queue_resources()
4799 	ctrl_info->max_hw_queue_index = num_queue_groups - 1;  in pqi_calculate_queue_resources()
4805 	ctrl_info->max_inbound_iu_length =  in pqi_calculate_queue_resources()
4806 		(ctrl_info->max_inbound_iu_length_per_firmware /  in pqi_calculate_queue_resources()
4811 		(ctrl_info->max_inbound_iu_length /  in pqi_calculate_queue_resources()
4818 		ctrl_info->max_elements_per_iq);  in pqi_calculate_queue_resources()
4822 		ctrl_info->max_elements_per_oq);  in pqi_calculate_queue_resources()
4824 	ctrl_info->num_elements_per_iq = num_elements_per_iq;  in pqi_calculate_queue_resources()
4825 	ctrl_info->num_elements_per_oq = num_elements_per_oq;  in pqi_calculate_queue_resources()
4827 	ctrl_info->max_sg_per_iu =  in pqi_calculate_queue_resources()
4828 		((ctrl_info->max_inbound_iu_length -  in pqi_calculate_queue_resources()
4845 static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_sg_list()  argument
4870 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_raid_sg_list()
4909 static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_sg_list()  argument
4935 	max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;  in pqi_build_aio_sg_list()
4986 	struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,  in pqi_raid_submit_scsi_cmd_with_io_request()  argument
5058 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_raid_submit_scsi_cmd_with_io_request()
5064 	rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
5070 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_scsi_cmd_with_io_request()
5075 static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_submit_scsi_cmd()  argument
5081 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_raid_submit_scsi_cmd()
5083 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_raid_submit_scsi_cmd()
5087 static inline void pqi_schedule_bypass_retry(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_bypass_retry()  argument
5089 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_schedule_bypass_retry()
5090 		schedule_work(&ctrl_info->raid_bypass_retry_work);  in pqi_schedule_bypass_retry()
5097 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_needed()  local
5112 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_raid_bypass_retry_needed()
5113 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_bypass_retry_needed()
5120 	struct pqi_ctrl_info *ctrl_info,  in pqi_add_to_raid_bypass_retry_list()  argument
5125 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
5128 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
5131 			&ctrl_info->raid_bypass_retry_list);  in pqi_add_to_raid_bypass_retry_list()
5132 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_add_to_raid_bypass_retry_list()
5148 	struct pqi_ctrl_info *ctrl_info;  in pqi_queue_raid_bypass_retry()  local
5153 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_queue_raid_bypass_retry()
5155 	pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request, false);  in pqi_queue_raid_bypass_retry()
5156 	pqi_schedule_bypass_retry(ctrl_info);  in pqi_queue_raid_bypass_retry()
5163 	struct pqi_ctrl_info *ctrl_info;  in pqi_retry_raid_bypass()  local
5175 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_retry_raid_bypass()
5180 	return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,  in pqi_retry_raid_bypass()
5185 	struct pqi_ctrl_info *ctrl_info)  in pqi_next_queued_raid_bypass_request()  argument
5190 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
5192 		&ctrl_info->raid_bypass_retry_list,  in pqi_next_queued_raid_bypass_request()
5196 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_next_queued_raid_bypass_request()
5201 static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_retry_raid_bypass_requests()  argument
5206 	pqi_ctrl_busy(ctrl_info);  in pqi_retry_raid_bypass_requests()
5209 		if (pqi_ctrl_blocked(ctrl_info))  in pqi_retry_raid_bypass_requests()
5211 		io_request = pqi_next_queued_raid_bypass_request(ctrl_info);  in pqi_retry_raid_bypass_requests()
5216 			pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request,  in pqi_retry_raid_bypass_requests()
5218 			pqi_schedule_bypass_retry(ctrl_info);  in pqi_retry_raid_bypass_requests()
5223 	pqi_ctrl_unbusy(ctrl_info);  in pqi_retry_raid_bypass_requests()
5228 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_worker()  local
5230 	ctrl_info = container_of(work, struct pqi_ctrl_info,  in pqi_raid_bypass_retry_worker()
5232 	pqi_retry_raid_bypass_requests(ctrl_info);  in pqi_raid_bypass_retry_worker()
5236 	struct pqi_ctrl_info *ctrl_info)  in pqi_clear_all_queued_raid_bypass_retries()  argument
5240 	spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
5241 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_clear_all_queued_raid_bypass_retries()
5242 	spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);  in pqi_clear_all_queued_raid_bypass_retries()
5262 static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_scsi_cmd()  argument
5266 	return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,  in pqi_aio_submit_scsi_cmd()
5270 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_io()  argument
5279 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_aio_submit_io()
5313 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_aio_submit_io()
5329 	rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);  in pqi_aio_submit_io()
5335 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
5340 static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,  in pqi_get_hw_queue()  argument
5346 	if (hw_queue > ctrl_info->max_hw_queue_index)  in pqi_get_hw_queue()
5379 	struct pqi_ctrl_info *ctrl_info;  in pqi_scsi_queue_command()  local
5386 	ctrl_info = shost_to_hba(shost);  in pqi_scsi_queue_command()
5396 	if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(ctrl_info,  in pqi_scsi_queue_command()
5403 	pqi_ctrl_busy(ctrl_info);  in pqi_scsi_queue_command()
5404 	if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device) ||  in pqi_scsi_queue_command()
5405 	    pqi_ctrl_in_ofa(ctrl_info)) {  in pqi_scsi_queue_command()
5416 	hw_queue = pqi_get_hw_queue(ctrl_info, scmd);  in pqi_scsi_queue_command()
5417 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
5423 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,  in pqi_scsi_queue_command()
5429 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5433 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5436 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,  in pqi_scsi_queue_command()
5441 	pqi_ctrl_unbusy(ctrl_info);  in pqi_scsi_queue_command()
5448 static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_until_queued_io_drained()  argument
5465 			pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_queued_io_drained()
5466 			if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_queued_io_drained()
5475 static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_inbound_queues_empty()  argument
5484 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_wait_until_inbound_queues_empty()
5485 		queue_group = &ctrl_info->queue_groups[i];  in pqi_wait_until_inbound_queues_empty()
5487 		rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);  in pqi_wait_until_inbound_queues_empty()
5498 				pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
5499 				if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_inbound_queues_empty()
5509 static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,  in pqi_fail_io_queued_for_device()  argument
5521 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_device()
5522 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
5550 static void pqi_fail_io_queued_for_all_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_io_queued_for_all_devices()  argument
5560 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_all_devices()
5561 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_all_devices()
5586 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_device_wait_for_pending_io()  argument
5594 		pqi_check_ctrl_health(ctrl_info);  in pqi_device_wait_for_pending_io()
5595 		if (pqi_ctrl_offline(ctrl_info))  in pqi_device_wait_for_pending_io()
5599 				dev_err(&ctrl_info->pci_dev->dev,  in pqi_device_wait_for_pending_io()
5610 static int pqi_ctrl_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_ctrl_wait_for_pending_io()  argument
5622 		spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ctrl_wait_for_pending_io()
5623 		list_for_each_entry(device, &ctrl_info->scsi_device_list,  in pqi_ctrl_wait_for_pending_io()
5630 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_ctrl_wait_for_pending_io()
5636 		pqi_check_ctrl_health(ctrl_info);  in pqi_ctrl_wait_for_pending_io()
5637 		if (pqi_ctrl_offline(ctrl_info))  in pqi_ctrl_wait_for_pending_io()
5642 				dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_wait_for_pending_io()
5663 static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_lun_reset_completion()  argument
5675 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_lun_reset_completion()
5676 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_lun_reset_completion()
5685 static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_lun_reset()  argument
5693 	io_request = pqi_alloc_io_request(ctrl_info);  in pqi_lun_reset()
5708 	pqi_start_io(ctrl_info,  in pqi_lun_reset()
5709 		&ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_lun_reset()
5712 	rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);  in pqi_lun_reset()
5727 static int _pqi_device_reset(struct pqi_ctrl_info *ctrl_info,  in _pqi_device_reset()  argument
5735 		rc = pqi_lun_reset(ctrl_info, device);  in _pqi_device_reset()
5743 	rc |= pqi_device_wait_for_pending_io(ctrl_info, device, timeout_secs);  in _pqi_device_reset()
5748 static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,  in pqi_device_reset()  argument
5753 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset()
5755 	pqi_ctrl_block_requests(ctrl_info);  in pqi_device_reset()
5756 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_device_reset()
5757 	pqi_fail_io_queued_for_device(ctrl_info, device);  in pqi_device_reset()
5758 	rc = pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_device_reset()
5760 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_device_reset()
5765 		rc = _pqi_device_reset(ctrl_info, device);  in pqi_device_reset()
5769 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset()
5778 	struct pqi_ctrl_info *ctrl_info;  in pqi_eh_device_reset_handler()  local
5782 	ctrl_info = shost_to_hba(shost);  in pqi_eh_device_reset_handler()
5785 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5789 	pqi_check_ctrl_health(ctrl_info);  in pqi_eh_device_reset_handler()
5790 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_eh_device_reset_handler()
5791 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5793 			ctrl_info->ctrl_id);  in pqi_eh_device_reset_handler()
5798 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_eh_device_reset_handler()
5800 	rc = pqi_device_reset(ctrl_info, device);  in pqi_eh_device_reset_handler()
5803 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_device_reset_handler()
5815 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_alloc()  local
5819 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_alloc()
5821 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5826 		device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);  in pqi_slave_alloc()
5833 		device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),  in pqi_slave_alloc()
5851 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
5858 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_map_queues()  local
5861 					ctrl_info->pci_dev, 0);  in pqi_map_queues()
5864 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info,  in pqi_getpciinfo_ioctl()  argument
5875 	pci_dev = ctrl_info->pci_dev;  in pqi_getpciinfo_ioctl()
5976 static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_passthru_ioctl()  argument
5987 	if (pqi_ctrl_offline(ctrl_info))  in pqi_passthru_ioctl()
6058 		rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_passthru_ioctl()
6069 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_passthru_ioctl()
6073 		pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_passthru_ioctl()
6118 	struct pqi_ctrl_info *ctrl_info;  in pqi_ioctl()  local
6120 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ioctl()
6122 	if (pqi_ctrl_in_ofa(ctrl_info))  in pqi_ioctl()
6129 		rc = pqi_scan_scsi_devices(ctrl_info);  in pqi_ioctl()
6132 		rc = pqi_getpciinfo_ioctl(ctrl_info, arg);  in pqi_ioctl()
6138 		rc = pqi_passthru_ioctl(ctrl_info, arg);  in pqi_ioctl()
6152 	struct pqi_ctrl_info *ctrl_info;  in pqi_firmware_version_show()  local
6155 	ctrl_info = shost_to_hba(shost);  in pqi_firmware_version_show()
6157 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);  in pqi_firmware_version_show()
6164 	struct pqi_ctrl_info *ctrl_info;  in pqi_driver_version_show()  local
6167 	ctrl_info = shost_to_hba(shost);  in pqi_driver_version_show()
6177 	struct pqi_ctrl_info *ctrl_info;  in pqi_serial_number_show()  local
6180 	ctrl_info = shost_to_hba(shost);  in pqi_serial_number_show()
6182 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);  in pqi_serial_number_show()
6189 	struct pqi_ctrl_info *ctrl_info;  in pqi_model_show()  local
6192 	ctrl_info = shost_to_hba(shost);  in pqi_model_show()
6194 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);  in pqi_model_show()
6201 	struct pqi_ctrl_info *ctrl_info;  in pqi_vendor_show()  local
6204 	ctrl_info = shost_to_hba(shost);  in pqi_vendor_show()
6206 	return snprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);  in pqi_vendor_show()
6282 	struct pqi_ctrl_info *ctrl_info;  in pqi_unique_id_show()  local
6289 	ctrl_info = shost_to_hba(sdev->host);  in pqi_unique_id_show()
6291 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
6295 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_unique_id_show()
6301 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
6314 	struct pqi_ctrl_info *ctrl_info;  in pqi_lunid_show()  local
6321 	ctrl_info = shost_to_hba(sdev->host);  in pqi_lunid_show()
6323 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
6327 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_lunid_show()
6333 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
6342 	struct pqi_ctrl_info *ctrl_info;  in pqi_path_info_show()  local
6355 	ctrl_info = shost_to_hba(sdev->host);  in pqi_path_info_show()
6357 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
6361 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_path_info_show()
6379 					ctrl_info->scsi_host->host_no,  in pqi_path_info_show()
6418 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
6426 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_address_show()  local
6433 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_address_show()
6435 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
6439 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,  in pqi_sas_address_show()
6445 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
6453 	struct pqi_ctrl_info *ctrl_info;  in pqi_ssd_smart_path_enabled_show()  local
6459 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ssd_smart_path_enabled_show()
6461 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
6468 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
6476 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_level_show()  local
6483 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_level_show()
6485 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
6494 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
6533 static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_register_scsi()  argument
6538 	shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));  in pqi_register_scsi()
6540 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6542 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6553 	shost->max_sectors = ctrl_info->max_sectors;  in pqi_register_scsi()
6554 	shost->can_queue = ctrl_info->scsi_ml_can_queue;  in pqi_register_scsi()
6556 	shost->sg_tablesize = ctrl_info->sg_tablesize;  in pqi_register_scsi()
6558 	shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);  in pqi_register_scsi()
6560 	shost->nr_hw_queues = ctrl_info->num_queue_groups;  in pqi_register_scsi()
6561 	shost->hostdata[0] = (unsigned long)ctrl_info;  in pqi_register_scsi()
6563 	rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);  in pqi_register_scsi()
6565 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6567 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6571 	rc = pqi_add_sas_host(shost, ctrl_info);  in pqi_register_scsi()
6573 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_register_scsi()
6575 			ctrl_info->ctrl_id);  in pqi_register_scsi()
6579 	ctrl_info->scsi_host = shost;  in pqi_register_scsi()
6591 static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_unregister_scsi()  argument
6595 	pqi_delete_sas_host(ctrl_info);  in pqi_unregister_scsi()
6597 	shost = ctrl_info->scsi_host;  in pqi_unregister_scsi()
6605 static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_reset_completion()  argument
6613 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_reset_completion()
6622 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_pqi_reset_completion()
6623 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_pqi_reset_completion()
6636 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_reset()  argument
6641 	if (ctrl_info->pqi_reset_quiesce_supported) {  in pqi_reset()
6642 		rc = sis_pqi_reset_quiesce(ctrl_info);  in pqi_reset()
6644 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
6655 	writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);  in pqi_reset()
6657 	rc = pqi_wait_for_pqi_reset_completion(ctrl_info);  in pqi_reset()
6659 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
6665 static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_serial_number()  argument
6674 	rc = pqi_sense_subsystem_info(ctrl_info, sense_info);  in pqi_get_ctrl_serial_number()
6678 	memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,  in pqi_get_ctrl_serial_number()
6680 	ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';  in pqi_get_ctrl_serial_number()
6688 static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_product_details()  argument
6697 	rc = pqi_identify_controller(ctrl_info, identify);  in pqi_get_ctrl_product_details()
6701 	memcpy(ctrl_info->firmware_version, identify->firmware_version,  in pqi_get_ctrl_product_details()
6703 	ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';  in pqi_get_ctrl_product_details()
6704 	snprintf(ctrl_info->firmware_version +  in pqi_get_ctrl_product_details()
6705 		strlen(ctrl_info->firmware_version),  in pqi_get_ctrl_product_details()
6706 		sizeof(ctrl_info->firmware_version),  in pqi_get_ctrl_product_details()
6709 	memcpy(ctrl_info->model, identify->product_id,  in pqi_get_ctrl_product_details()
6711 	ctrl_info->model[sizeof(identify->product_id)] = '\0';  in pqi_get_ctrl_product_details()
6713 	memcpy(ctrl_info->vendor, identify->vendor_id,  in pqi_get_ctrl_product_details()
6715 	ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';  in pqi_get_ctrl_product_details()
6724 	struct pqi_ctrl_info *ctrl_info;  member
6777 static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,  in pqi_config_table_update()  argument
6794 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_config_table_update()
6798 static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,  in pqi_enable_firmware_features()  argument
6814 	return pqi_config_table_update(ctrl_info,  in pqi_enable_firmware_features()
6824 	void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
6828 static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_status()  argument
6832 		dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",  in pqi_firmware_feature_status()
6838 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_firmware_feature_status()
6843 	dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",  in pqi_firmware_feature_status()
6847 static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_update()  argument
6851 		firmware_feature->feature_status(ctrl_info, firmware_feature);  in pqi_firmware_feature_update()
6878 	struct pqi_ctrl_info *ctrl_info;  in pqi_process_firmware_features()  local
6884 	ctrl_info = section_info->ctrl_info;  in pqi_process_firmware_features()
6895 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
6910 	rc = pqi_enable_firmware_features(ctrl_info, firmware_features,  in pqi_process_firmware_features()
6913 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_firmware_features()
6918 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
6924 	ctrl_info->soft_reset_handshake_supported = false;  in pqi_process_firmware_features()
6934 				ctrl_info->soft_reset_handshake_supported =  in pqi_process_firmware_features()
6937 		pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
6961 static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)  in pqi_process_config_table()  argument
6970 	table_length = ctrl_info->config_table_length;  in pqi_process_config_table()
6976 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
6985 	table_iomem_addr = ctrl_info->iomem_base +  in pqi_process_config_table()
6986 		ctrl_info->config_table_offset;  in pqi_process_config_table()
6989 	section_info.ctrl_info = ctrl_info;  in pqi_process_config_table()
7007 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
7010 				ctrl_info->heartbeat_counter =  in pqi_process_config_table()
7018 			ctrl_info->soft_reset_status =  in pqi_process_config_table()
7037 static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_revert_to_sis_mode()  argument
7041 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);  in pqi_revert_to_sis_mode()
7042 	rc = pqi_reset(ctrl_info);  in pqi_revert_to_sis_mode()
7045 	rc = sis_reenable_sis_mode(ctrl_info);  in pqi_revert_to_sis_mode()
7047 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_revert_to_sis_mode()
7051 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_revert_to_sis_mode()
7061 static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_force_sis_mode()  argument
7063 	if (!sis_is_firmware_running(ctrl_info))  in pqi_force_sis_mode()
7066 	if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)  in pqi_force_sis_mode()
7069 	if (sis_is_kernel_up(ctrl_info)) {  in pqi_force_sis_mode()
7070 		pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_force_sis_mode()
7074 	return pqi_revert_to_sis_mode(ctrl_info);  in pqi_force_sis_mode()
7077 static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init()  argument
7081 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init()
7089 	rc = sis_wait_for_ctrl_ready(ctrl_info);  in pqi_ctrl_init()
7097 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init()
7099 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7104 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init()
7106 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7112 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
7114 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
7117 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
7119 			ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
7123 	pqi_calculate_io_resources(ctrl_info);  in pqi_ctrl_init()
7125 	rc = pqi_alloc_error_buffer(ctrl_info);  in pqi_ctrl_init()
7127 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7137 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init()
7139 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7145 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init()
7147 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7153 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init()
7154 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init()
7156 	rc = pqi_alloc_admin_queues(ctrl_info);  in pqi_ctrl_init()
7158 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7163 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init()
7165 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7170 	rc = pqi_report_device_capability(ctrl_info);  in pqi_ctrl_init()
7172 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7177 	rc = pqi_validate_device_capability(ctrl_info);  in pqi_ctrl_init()
7181 	pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
7183 	rc = pqi_enable_msix_interrupts(ctrl_info);  in pqi_ctrl_init()
7187 	if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {  in pqi_ctrl_init()
7188 		ctrl_info->max_msix_vectors =  in pqi_ctrl_init()
7189 			ctrl_info->num_msix_vectors_enabled;  in pqi_ctrl_init()
7190 		pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
7193 	rc = pqi_alloc_io_resources(ctrl_info);  in pqi_ctrl_init()
7197 	rc = pqi_alloc_operational_queues(ctrl_info);  in pqi_ctrl_init()
7199 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7204 	pqi_init_operational_queues(ctrl_info);  in pqi_ctrl_init()
7206 	rc = pqi_request_irqs(ctrl_info);  in pqi_ctrl_init()
7210 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init()
7214 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init()
7216 	ctrl_info->controller_online = true;  in pqi_ctrl_init()
7218 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init()
7222 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init()
7224 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init()
7226 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7232 	rc = pqi_register_scsi(ctrl_info);  in pqi_ctrl_init()
7236 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init()
7238 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7243 	rc = pqi_get_ctrl_serial_number(ctrl_info);  in pqi_ctrl_init()
7245 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7250 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init()
7252 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7257 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init()
7259 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
7264 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init()
7266 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init()
7271 static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_reinit_queues()  argument
7277 	admin_queues = &ctrl_info->admin_queues;  in pqi_reinit_queues()
7282 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_reinit_queues()
7283 		ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;  in pqi_reinit_queues()
7284 		ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;  in pqi_reinit_queues()
7285 		ctrl_info->queue_groups[i].oq_ci_copy = 0;  in pqi_reinit_queues()
7287 		writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);  in pqi_reinit_queues()
7288 		writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);  in pqi_reinit_queues()
7289 		writel(0, ctrl_info->queue_groups[i].oq_pi);  in pqi_reinit_queues()
7292 	event_queue = &ctrl_info->event_queue;  in pqi_reinit_queues()
7297 static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init_resume()  argument
7301 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init_resume()
7309 	rc = sis_wait_for_ctrl_ready_resume(ctrl_info);  in pqi_ctrl_init_resume()
7317 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init_resume()
7319 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7324 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init_resume()
7326 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7336 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init_resume()
7338 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7344 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init_resume()
7346 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7352 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init_resume()
7353 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init_resume()
7355 	pqi_reinit_queues(ctrl_info);  in pqi_ctrl_init_resume()
7357 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init_resume()
7359 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7364 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init_resume()
7368 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init_resume()
7370 	ctrl_info->controller_online = true;  in pqi_ctrl_init_resume()
7371 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ctrl_init_resume()
7373 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init_resume()
7377 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init_resume()
7379 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init_resume()
7381 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7386 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init_resume()
7388 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7393 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init_resume()
7395 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7400 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init_resume()
7402 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
7407 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init_resume()
7409 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init_resume()
7421 static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_pci_init()  argument
7426 	rc = pci_enable_device(ctrl_info->pci_dev);  in pqi_pci_init()
7428 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7438 	rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);  in pqi_pci_init()
7440 		dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");  in pqi_pci_init()
7444 	rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);  in pqi_pci_init()
7446 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7451 	ctrl_info->iomem_base = ioremap_nocache(pci_resource_start(  in pqi_pci_init()
7452 		ctrl_info->pci_dev, 0),  in pqi_pci_init()
7454 	if (!ctrl_info->iomem_base) {  in pqi_pci_init()
7455 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7464 	rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,  in pqi_pci_init()
7467 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
7473 	pci_set_master(ctrl_info->pci_dev);  in pqi_pci_init()
7475 	ctrl_info->registers = ctrl_info->iomem_base;  in pqi_pci_init()
7476 	ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;  in pqi_pci_init()
7478 	pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);  in pqi_pci_init()
7483 	pci_release_regions(ctrl_info->pci_dev);  in pqi_pci_init()
7485 	pci_disable_device(ctrl_info->pci_dev);  in pqi_pci_init()
7490 static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_cleanup_pci_init()  argument
7492 	iounmap(ctrl_info->iomem_base);  in pqi_cleanup_pci_init()
7493 	pci_release_regions(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
7494 	if (pci_is_enabled(ctrl_info->pci_dev))  in pqi_cleanup_pci_init()
7495 		pci_disable_device(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
7496 	pci_set_drvdata(ctrl_info->pci_dev, NULL);  in pqi_cleanup_pci_init()
7501 	struct pqi_ctrl_info *ctrl_info;  in pqi_alloc_ctrl_info()  local
7503 	ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),  in pqi_alloc_ctrl_info()
7505 	if (!ctrl_info)  in pqi_alloc_ctrl_info()
7508 	mutex_init(&ctrl_info->scan_mutex);  in pqi_alloc_ctrl_info()
7509 	mutex_init(&ctrl_info->lun_reset_mutex);  in pqi_alloc_ctrl_info()
7510 	mutex_init(&ctrl_info->ofa_mutex);  in pqi_alloc_ctrl_info()
7512 	INIT_LIST_HEAD(&ctrl_info->scsi_device_list);  in pqi_alloc_ctrl_info()
7513 	spin_lock_init(&ctrl_info->scsi_device_list_lock);  in pqi_alloc_ctrl_info()
7515 	INIT_WORK(&ctrl_info->event_work, pqi_event_worker);  in pqi_alloc_ctrl_info()
7516 	atomic_set(&ctrl_info->num_interrupts, 0);  in pqi_alloc_ctrl_info()
7518 	INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);  in pqi_alloc_ctrl_info()
7519 	INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);  in pqi_alloc_ctrl_info()
7521 	timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);  in pqi_alloc_ctrl_info()
7522 	INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);  in pqi_alloc_ctrl_info()
7524 	sema_init(&ctrl_info->sync_request_sem,  in pqi_alloc_ctrl_info()
7526 	init_waitqueue_head(&ctrl_info->block_requests_wait);  in pqi_alloc_ctrl_info()
7528 	INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);  in pqi_alloc_ctrl_info()
7529 	spin_lock_init(&ctrl_info->raid_bypass_retry_list_lock);  in pqi_alloc_ctrl_info()
7530 	INIT_WORK(&ctrl_info->raid_bypass_retry_work,  in pqi_alloc_ctrl_info()
7533 	ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;  in pqi_alloc_ctrl_info()
7534 	ctrl_info->irq_mode = IRQ_MODE_NONE;  in pqi_alloc_ctrl_info()
7535 	ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;  in pqi_alloc_ctrl_info()
7537 	return ctrl_info;  in pqi_alloc_ctrl_info()
7540 static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_info()  argument
7542 	kfree(ctrl_info);  in pqi_free_ctrl_info()
7545 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_free_interrupts()  argument
7547 	pqi_free_irqs(ctrl_info);  in pqi_free_interrupts()
7548 	pqi_disable_msix_interrupts(ctrl_info);  in pqi_free_interrupts()
7551 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_resources()  argument
7553 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_free_ctrl_resources()
7554 	pqi_free_interrupts(ctrl_info);  in pqi_free_ctrl_resources()
7555 	if (ctrl_info->queue_memory_base)  in pqi_free_ctrl_resources()
7556 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7557 			ctrl_info->queue_memory_length,  in pqi_free_ctrl_resources()
7558 			ctrl_info->queue_memory_base,  in pqi_free_ctrl_resources()
7559 			ctrl_info->queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
7560 	if (ctrl_info->admin_queue_memory_base)  in pqi_free_ctrl_resources()
7561 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7562 			ctrl_info->admin_queue_memory_length,  in pqi_free_ctrl_resources()
7563 			ctrl_info->admin_queue_memory_base,  in pqi_free_ctrl_resources()
7564 			ctrl_info->admin_queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
7565 	pqi_free_all_io_requests(ctrl_info);  in pqi_free_ctrl_resources()
7566 	if (ctrl_info->error_buffer)  in pqi_free_ctrl_resources()
7567 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
7568 			ctrl_info->error_buffer_length,  in pqi_free_ctrl_resources()
7569 			ctrl_info->error_buffer,  in pqi_free_ctrl_resources()
7570 			ctrl_info->error_buffer_dma_handle);  in pqi_free_ctrl_resources()
7571 	if (ctrl_info->iomem_base)  in pqi_free_ctrl_resources()
7572 		pqi_cleanup_pci_init(ctrl_info);  in pqi_free_ctrl_resources()
7573 	pqi_free_ctrl_info(ctrl_info);  in pqi_free_ctrl_resources()
7576 static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_ctrl()  argument
7578 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_remove_ctrl()
7579 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_remove_ctrl()
7580 	pqi_remove_all_scsi_devices(ctrl_info);  in pqi_remove_ctrl()
7581 	pqi_unregister_scsi(ctrl_info);  in pqi_remove_ctrl()
7582 	if (ctrl_info->pqi_mode_enabled)  in pqi_remove_ctrl()
7583 		pqi_revert_to_sis_mode(ctrl_info);  in pqi_remove_ctrl()
7584 	pqi_free_ctrl_resources(ctrl_info);  in pqi_remove_ctrl()
7587 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_quiesce()  argument
7589 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7590 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7591 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7592 	pqi_wait_until_scan_finished(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7593 	pqi_ctrl_ofa_start(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7594 	pqi_ctrl_block_requests(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7595 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7596 	pqi_ctrl_wait_for_pending_io(ctrl_info, PQI_PENDING_IO_TIMEOUT_SECS);  in pqi_ofa_ctrl_quiesce()
7597 	pqi_fail_io_queued_for_all_devices(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7598 	pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7599 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_quiesce()
7600 	ctrl_info->pqi_mode_enabled = false;  in pqi_ofa_ctrl_quiesce()
7601 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_ofa_ctrl_quiesce()
7604 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_unquiesce()  argument
7606 	pqi_ofa_free_host_buffer(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7607 	ctrl_info->pqi_mode_enabled = true;  in pqi_ofa_ctrl_unquiesce()
7608 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ofa_ctrl_unquiesce()
7609 	ctrl_info->controller_online = true;  in pqi_ofa_ctrl_unquiesce()
7610 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7611 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7612 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7613 	pqi_clear_soft_reset_status(ctrl_info,  in pqi_ofa_ctrl_unquiesce()
7615 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
7618 static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_alloc_mem()  argument
7628 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_alloc_mem()
7633 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_alloc_mem()
7638 	ctrl_info->pqi_ofa_chunk_virt_addr =  in pqi_ofa_alloc_mem()
7640 	if (!ctrl_info->pqi_ofa_chunk_virt_addr)  in pqi_ofa_alloc_mem()
7646 		ctrl_info->pqi_ofa_chunk_virt_addr[i] =  in pqi_ofa_alloc_mem()
7650 		if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])  in pqi_ofa_alloc_mem()
7671 				ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_alloc_mem()
7674 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_alloc_mem()
7681 static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_alloc_host_buffer()  argument
7688 			ctrl_info->pqi_ofa_mem_virt_addr->bytes_allocated);  in pqi_ofa_alloc_host_buffer()
7692 		if (!pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_sz))  in pqi_ofa_alloc_host_buffer()
7698 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_setup_host_buffer()  argument
7704 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_setup_host_buffer()
7707 					    &ctrl_info->pqi_ofa_mem_dma_handle,  in pqi_ofa_setup_host_buffer()
7718 	ctrl_info->pqi_ofa_mem_virt_addr = pqi_ofa_memory;  in pqi_ofa_setup_host_buffer()
7720 	if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {  in pqi_ofa_setup_host_buffer()
7726 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_free_host_buffer()  argument
7732 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_free_host_buffer()
7744 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_ofa_free_host_buffer()
7746 			ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_free_host_buffer()
7749 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_free_host_buffer()
7752 	dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_ofa_free_host_buffer()
7754 			ctrl_info->pqi_ofa_mem_dma_handle);  in pqi_ofa_free_host_buffer()
7755 	ctrl_info->pqi_ofa_mem_virt_addr = NULL;  in pqi_ofa_free_host_buffer()
7758 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_host_memory_update()  argument
7766 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_host_memory_update()
7779 		put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,  in pqi_ofa_host_memory_update()
7786 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_ofa_host_memory_update()
7792 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_restart()  argument
7795 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_ofa_ctrl_restart()
7818 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_all_outstanding_requests()  argument
7824 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_fail_all_outstanding_requests()
7825 		io_request = &ctrl_info->io_request_pool[i];  in pqi_fail_all_outstanding_requests()
7843 static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline_deferred()  argument
7846 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7847 	pqi_free_interrupts(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7848 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7849 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7850 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7851 	pqi_fail_all_outstanding_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7852 	pqi_clear_all_queued_raid_bypass_retries(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7853 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
7858 	struct pqi_ctrl_info *ctrl_info;  in pqi_ctrl_offline_worker()  local
7860 	ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);  in pqi_ctrl_offline_worker()
7861 	pqi_take_ctrl_offline_deferred(ctrl_info);  in pqi_ctrl_offline_worker()
7864 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline()  argument
7866 	if (!ctrl_info->controller_online)  in pqi_take_ctrl_offline()
7869 	ctrl_info->controller_online = false;  in pqi_take_ctrl_offline()
7870 	ctrl_info->pqi_mode_enabled = false;  in pqi_take_ctrl_offline()
7871 	pqi_ctrl_block_requests(ctrl_info);  in pqi_take_ctrl_offline()
7873 		sis_shutdown_ctrl(ctrl_info);  in pqi_take_ctrl_offline()
7874 	pci_disable_device(ctrl_info->pci_dev);  in pqi_take_ctrl_offline()
7875 	dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");  in pqi_take_ctrl_offline()
7876 	schedule_work(&ctrl_info->ctrl_offline_work);  in pqi_take_ctrl_offline()
7897 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_probe()  local
7921 	ctrl_info = pqi_alloc_ctrl_info(node);  in pqi_pci_probe()
7922 	if (!ctrl_info) {  in pqi_pci_probe()
7928 	ctrl_info->pci_dev = pci_dev;  in pqi_pci_probe()
7930 	rc = pqi_pci_init(ctrl_info);  in pqi_pci_probe()
7934 	rc = pqi_ctrl_init(ctrl_info);  in pqi_pci_probe()
7941 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_probe()
7948 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_remove()  local
7950 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_pci_remove()
7951 	if (!ctrl_info)  in pqi_pci_remove()
7954 	ctrl_info->in_shutdown = true;  in pqi_pci_remove()
7956 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_remove()
7962 	struct pqi_ctrl_info *ctrl_info;  in pqi_shutdown()  local
7964 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_shutdown()
7965 	if (!ctrl_info)  in pqi_shutdown()
7972 	rc = pqi_flush_cache(ctrl_info, SHUTDOWN);  in pqi_shutdown()
7973 	pqi_free_interrupts(ctrl_info);  in pqi_shutdown()
7974 	pqi_reset(ctrl_info);  in pqi_shutdown()
8009 	struct pqi_ctrl_info *ctrl_info;  in pqi_suspend()  local
8011 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_suspend()
8013 	pqi_disable_events(ctrl_info);  in pqi_suspend()
8014 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_suspend()
8015 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_suspend()
8016 	pqi_wait_until_scan_finished(ctrl_info);  in pqi_suspend()
8017 	pqi_wait_until_lun_reset_finished(ctrl_info);  in pqi_suspend()
8018 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_suspend()
8019 	pqi_flush_cache(ctrl_info, SUSPEND);  in pqi_suspend()
8020 	pqi_ctrl_block_requests(ctrl_info);  in pqi_suspend()
8021 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_suspend()
8022 	pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_suspend()
8023 	pqi_ctrl_wait_for_pending_io(ctrl_info, NO_TIMEOUT);  in pqi_suspend()
8024 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_suspend()
8032 	ctrl_info->controller_online = false;  in pqi_suspend()
8033 	ctrl_info->pqi_mode_enabled = false;  in pqi_suspend()
8041 	struct pqi_ctrl_info *ctrl_info;  in pqi_resume()  local
8043 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_resume()
8046 		ctrl_info->max_hw_queue_index = 0;  in pqi_resume()
8047 		pqi_free_interrupts(ctrl_info);  in pqi_resume()
8048 		pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);  in pqi_resume()
8051 			&ctrl_info->queue_groups[0]);  in pqi_resume()
8053 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_resume()
8058 		pqi_start_heartbeat_timer(ctrl_info);  in pqi_resume()
8059 		pqi_ctrl_unblock_requests(ctrl_info);  in pqi_resume()
8066 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_resume()