69 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,
72 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
74 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
77 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
80 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
84 static  int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,
88 static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,
92 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info);
93 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info);
94 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs);
95 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info);
96 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info);
97 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info);
98 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
100 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info);
247 static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_offline()  argument
249 	return !ctrl_info->controller_online;  in pqi_ctrl_offline()
252 static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)  in pqi_check_ctrl_health()  argument
254 	if (ctrl_info->controller_online)  in pqi_check_ctrl_health()
255 		if (!sis_is_firmware_running(ctrl_info))  in pqi_check_ctrl_health()
256 			pqi_take_ctrl_offline(ctrl_info, PQI_FIRMWARE_KERNEL_NOT_UP);  in pqi_check_ctrl_health()
267 static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_mode()  argument
269 	return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;  in pqi_get_ctrl_mode()
272 static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_save_ctrl_mode()  argument
277 	driver_scratch = sis_read_driver_scratch(ctrl_info);  in pqi_save_ctrl_mode()
284 	sis_write_driver_scratch(ctrl_info, driver_scratch);  in pqi_save_ctrl_mode()
287 static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)  in pqi_is_fw_triage_supported()  argument
289 	return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;  in pqi_is_fw_triage_supported()
292 static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)  in pqi_save_fw_triage_setting()  argument
296 	driver_scratch = sis_read_driver_scratch(ctrl_info);  in pqi_save_fw_triage_setting()
303 	sis_write_driver_scratch(ctrl_info, driver_scratch);  in pqi_save_fw_triage_setting()
306 static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_scan()  argument
308 	ctrl_info->scan_blocked = true;  in pqi_ctrl_block_scan()
309 	mutex_lock(&ctrl_info->scan_mutex);  in pqi_ctrl_block_scan()
312 static inline void pqi_ctrl_unblock_scan(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_scan()  argument
314 	ctrl_info->scan_blocked = false;  in pqi_ctrl_unblock_scan()
315 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_ctrl_unblock_scan()
318 static inline bool pqi_ctrl_scan_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_scan_blocked()  argument
320 	return ctrl_info->scan_blocked;  in pqi_ctrl_scan_blocked()
323 static inline void pqi_ctrl_block_device_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_device_reset()  argument
325 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_ctrl_block_device_reset()
328 static inline void pqi_ctrl_unblock_device_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_device_reset()  argument
330 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_ctrl_unblock_device_reset()
333 static inline void pqi_scsi_block_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_scsi_block_requests()  argument
339 	shost = ctrl_info->scsi_host;  in pqi_scsi_block_requests()
353 static inline void pqi_scsi_unblock_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_scsi_unblock_requests()  argument
355 	scsi_unblock_requests(ctrl_info->scsi_host);  in pqi_scsi_unblock_requests()
358 static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_busy()  argument
360 	atomic_inc(&ctrl_info->num_busy_threads);  in pqi_ctrl_busy()
363 static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unbusy()  argument
365 	atomic_dec(&ctrl_info->num_busy_threads);  in pqi_ctrl_unbusy()
368 static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_blocked()  argument
370 	return ctrl_info->block_requests;  in pqi_ctrl_blocked()
373 static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_block_requests()  argument
375 	ctrl_info->block_requests = true;  in pqi_ctrl_block_requests()
378 static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_unblock_requests()  argument
380 	ctrl_info->block_requests = false;  in pqi_ctrl_unblock_requests()
381 	wake_up_all(&ctrl_info->block_requests_wait);  in pqi_ctrl_unblock_requests()
384 static void pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_if_ctrl_blocked()  argument
386 	if (!pqi_ctrl_blocked(ctrl_info))  in pqi_wait_if_ctrl_blocked()
389 	atomic_inc(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
390 	wait_event(ctrl_info->block_requests_wait,  in pqi_wait_if_ctrl_blocked()
391 		!pqi_ctrl_blocked(ctrl_info));  in pqi_wait_if_ctrl_blocked()
392 	atomic_dec(&ctrl_info->num_blocked_threads);  in pqi_wait_if_ctrl_blocked()
397 static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_wait_until_quiesced()  argument
407 	while (atomic_read(&ctrl_info->num_busy_threads) >  in pqi_ctrl_wait_until_quiesced()
408 		atomic_read(&ctrl_info->num_blocked_threads)) {  in pqi_ctrl_wait_until_quiesced()
410 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_ctrl_wait_until_quiesced()
420 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_ctrl_wait_until_quiesced()
430 static inline void pqi_ctrl_ofa_start(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_start()  argument
432 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_ctrl_ofa_start()
435 static inline void pqi_ctrl_ofa_done(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_ofa_done()  argument
437 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_ctrl_ofa_done()
440 static inline void pqi_wait_until_ofa_finished(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_ofa_finished()  argument
442 	mutex_lock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
443 	mutex_unlock(&ctrl_info->ofa_mutex);  in pqi_wait_until_ofa_finished()
446 static inline bool pqi_ofa_in_progress(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_in_progress()  argument
448 	return mutex_is_locked(&ctrl_info->ofa_mutex);  in pqi_ofa_in_progress()
492 static inline void pqi_schedule_rescan_worker_with_delay(struct pqi_ctrl_info *ctrl_info,  in pqi_schedule_rescan_worker_with_delay()  argument
495 	if (pqi_ctrl_offline(ctrl_info))  in pqi_schedule_rescan_worker_with_delay()
498 	schedule_delayed_work(&ctrl_info->rescan_work, delay);  in pqi_schedule_rescan_worker_with_delay()
501 static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker()  argument
503 	pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);  in pqi_schedule_rescan_worker()
508 static inline void pqi_schedule_rescan_worker_delayed(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_rescan_worker_delayed()  argument
510 	pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);  in pqi_schedule_rescan_worker_delayed()
513 static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_rescan_worker()  argument
515 	cancel_delayed_work_sync(&ctrl_info->rescan_work);  in pqi_cancel_rescan_worker()
518 static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)  in pqi_read_heartbeat_counter()  argument
520 	if (!ctrl_info->heartbeat_counter)  in pqi_read_heartbeat_counter()
523 	return readl(ctrl_info->heartbeat_counter);  in pqi_read_heartbeat_counter()
526 static inline u8 pqi_read_soft_reset_status(struct pqi_ctrl_info *ctrl_info)  in pqi_read_soft_reset_status()  argument
528 	return readb(ctrl_info->soft_reset_status);  in pqi_read_soft_reset_status()
531 static inline void pqi_clear_soft_reset_status(struct pqi_ctrl_info *ctrl_info)  in pqi_clear_soft_reset_status()  argument
535 	status = pqi_read_soft_reset_status(ctrl_info);  in pqi_clear_soft_reset_status()
537 	writeb(status, ctrl_info->soft_reset_status);  in pqi_clear_soft_reset_status()
607 static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_path_request()  argument
643 			if (ctrl_info->rpl_extended_format_4_5_supported)  in pqi_build_raid_path_request()
648 			cdb[1] = ctrl_info->ciss_report_log_flags;  in pqi_build_raid_path_request()
694 		dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n", cmd);  in pqi_build_raid_path_request()
713 	return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],  in pqi_build_raid_path_request()
725 static inline struct pqi_io_request *pqi_alloc_io_request(struct pqi_ctrl_info *ctrl_info, struct s…  in pqi_alloc_io_request()  argument
734 		io_request = &ctrl_info->io_request_pool[i];  in pqi_alloc_io_request()
747 			io_request = &ctrl_info->io_request_pool[ctrl_info->scsi_ml_can_queue + i];  in pqi_alloc_io_request()
766 static int pqi_send_scsi_raid_request(struct pqi_ctrl_info *ctrl_info, u8 cmd,  in pqi_send_scsi_raid_request()  argument
774 	rc = pqi_build_raid_path_request(ctrl_info, &request, cmd, scsi3addr,  in pqi_send_scsi_raid_request()
779 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, error_info);  in pqi_send_scsi_raid_request()
781 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_send_scsi_raid_request()
788 static inline int pqi_send_ctrl_raid_request(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_request()  argument
791 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_request()
795 static inline int pqi_send_ctrl_raid_with_error(struct pqi_ctrl_info *ctrl_info,  in pqi_send_ctrl_raid_with_error()  argument
799 	return pqi_send_scsi_raid_request(ctrl_info, cmd, RAID_CTLR_LUNID,  in pqi_send_ctrl_raid_with_error()
803 static inline int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_controller()  argument
806 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_IDENTIFY_CONTROLLER,  in pqi_identify_controller()
810 static inline int pqi_sense_subsystem_info(struct  pqi_ctrl_info *ctrl_info,  in pqi_sense_subsystem_info()  argument
813 	return pqi_send_ctrl_raid_request(ctrl_info,  in pqi_sense_subsystem_info()
818 static inline int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_inquiry()  argument
821 	return pqi_send_scsi_raid_request(ctrl_info, INQUIRY, scsi3addr,  in pqi_scsi_inquiry()
825 static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,  in pqi_identify_physical_device()  argument
834 	rc = pqi_build_raid_path_request(ctrl_info, &request,  in pqi_identify_physical_device()
844 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_identify_physical_device()
846 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_identify_physical_device()
882 static int pqi_get_advanced_raid_bypass_config(struct pqi_ctrl_info *ctrl_info)  in pqi_get_advanced_raid_bypass_config()  argument
893 	rc = pqi_build_raid_path_request(ctrl_info, &request, BMIC_SENSE_FEATURE, RAID_CTLR_LUNID,  in pqi_get_advanced_raid_bypass_config()
901 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_get_advanced_raid_bypass_config()
903 	pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1, dir);  in pqi_get_advanced_raid_bypass_config()
922 	ctrl_info->max_transfer_encrypted_sas_sata =  in pqi_get_advanced_raid_bypass_config()
926 	ctrl_info->max_transfer_encrypted_nvme =  in pqi_get_advanced_raid_bypass_config()
930 	ctrl_info->max_write_raid_5_6 =  in pqi_get_advanced_raid_bypass_config()
934 	ctrl_info->max_write_raid_1_10_2drive =  in pqi_get_advanced_raid_bypass_config()
938 	ctrl_info->max_write_raid_1_10_3drive =  in pqi_get_advanced_raid_bypass_config()
948 static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info,  in pqi_flush_cache()  argument
960 	rc = pqi_send_ctrl_raid_request(ctrl_info, SA_FLUSH_CACHE, flush_cache,  in pqi_flush_cache()
968 int pqi_csmi_smp_passthru(struct pqi_ctrl_info *ctrl_info,  in pqi_csmi_smp_passthru()  argument
972 	return pqi_send_ctrl_raid_with_error(ctrl_info, BMIC_CSMI_PASSTHRU,  in pqi_csmi_smp_passthru()
978 static int pqi_set_diag_rescan(struct pqi_ctrl_info *ctrl_info)  in pqi_set_diag_rescan()  argument
987 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SENSE_DIAG_OPTIONS,  in pqi_set_diag_rescan()
994 	rc = pqi_send_ctrl_raid_request(ctrl_info, BMIC_SET_DIAG_OPTIONS, diag,  in pqi_set_diag_rescan()
1003 static inline int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,  in pqi_write_host_wellness()  argument
1006 	return pqi_send_ctrl_raid_request(ctrl_info, BMIC_WRITE_HOST_WELLNESS,  in pqi_write_host_wellness()
1024 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_driver_version_to_host_wellness()  argument
1052 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_driver_version_to_host_wellness()
1073 	struct pqi_ctrl_info *ctrl_info)  in pqi_write_current_time_to_host_wellness()  argument
1115 	rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);  in pqi_write_current_time_to_host_wellness()
1127 	struct pqi_ctrl_info *ctrl_info;  in pqi_update_time_worker()  local
1129 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_update_time_worker()
1132 	rc = pqi_write_current_time_to_host_wellness(ctrl_info);  in pqi_update_time_worker()
1134 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_time_worker()
1137 	schedule_delayed_work(&ctrl_info->update_time_work,  in pqi_update_time_worker()
1141 static inline void pqi_schedule_update_time_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_schedule_update_time_worker()  argument
1143 	schedule_delayed_work(&ctrl_info->update_time_work, 0);  in pqi_schedule_update_time_worker()
1146 static inline void pqi_cancel_update_time_worker(struct pqi_ctrl_info *ctrl_info)  in pqi_cancel_update_time_worker()  argument
1148 	cancel_delayed_work_sync(&ctrl_info->update_time_work);  in pqi_cancel_update_time_worker()
1151 static inline int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void *buffer,  in pqi_report_luns()  argument
1154 	return pqi_send_ctrl_raid_request(ctrl_info, cmd, buffer, buffer_length);  in pqi_report_luns()
1157 static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd, void **buffer)  in pqi_report_phys_logical_luns()  argument
1172 	rc = pqi_report_luns(ctrl_info, cmd, report_lun_header, sizeof(*report_lun_header));  in pqi_report_phys_logical_luns()
1192 	rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);  in pqi_report_phys_logical_luns()
1218 static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)  in pqi_report_phys_luns()  argument
1229 	rc = pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS, &rpl_list);  in pqi_report_phys_luns()
1233 	if (ctrl_info->rpl_extended_format_4_5_supported) {  in pqi_report_phys_luns()
1240 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_report_phys_luns()
1245 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_report_phys_luns()
1279 static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info, void **buffer)  in pqi_report_logical_luns()  argument
1281 	return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);  in pqi_report_logical_luns()
1284 static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_lists()  argument
1295 	rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);  in pqi_get_device_lists()
1297 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1300 	rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);  in pqi_get_device_lists()
1302 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_get_device_lists()
1394 static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_level()  argument
1405 		rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_level()
1418 static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_validate_raid_map()  argument
1458 	dev_warn(&ctrl_info->pci_dev->dev,  in pqi_validate_raid_map()
1466 static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_map()  argument
1477 	rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1492 		rc = pqi_send_scsi_raid_request(ctrl_info, CISS_GET_RAID_MAP,  in pqi_get_raid_map()
1499 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_get_raid_map()
1508 	rc = pqi_validate_raid_map(ctrl_info, device, raid_map);  in pqi_get_raid_map()
1522 static void pqi_set_max_transfer_encrypted(struct pqi_ctrl_info *ctrl_info,  in pqi_set_max_transfer_encrypted()  argument
1525 	if (!ctrl_info->lv_drive_type_mix_valid) {  in pqi_set_max_transfer_encrypted()
1539 			ctrl_info->max_transfer_encrypted_sas_sata;  in pqi_set_max_transfer_encrypted()
1543 			ctrl_info->max_transfer_encrypted_nvme;  in pqi_set_max_transfer_encrypted()
1549 			min(ctrl_info->max_transfer_encrypted_sas_sata,  in pqi_set_max_transfer_encrypted()
1550 				ctrl_info->max_transfer_encrypted_nvme);  in pqi_set_max_transfer_encrypted()
1555 static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_raid_bypass_status()  argument
1566 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_raid_bypass_status()
1580 		pqi_get_raid_map(ctrl_info, device) == 0) {  in pqi_get_raid_bypass_status()
1584 			pqi_set_max_transfer_encrypted(ctrl_info, device);  in pqi_get_raid_bypass_status()
1595 static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_get_volume_status()  argument
1609 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,  in pqi_get_volume_status()
1637 static int pqi_get_physical_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_physical_device_info()  argument
1645 	rc = pqi_identify_physical_device(ctrl_info, device,  in pqi_get_physical_device_info()
1689 static int pqi_get_logical_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_logical_device_info()  argument
1700 	rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);  in pqi_get_logical_device_info()
1717 			pqi_get_raid_level(ctrl_info, device);  in pqi_get_logical_device_info()
1718 			pqi_get_raid_bypass_status(ctrl_info, device);  in pqi_get_logical_device_info()
1719 			pqi_get_volume_status(ctrl_info, device);  in pqi_get_logical_device_info()
1744 static int pqi_get_device_info_phys_logical(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_info_phys_logical()  argument
1754 		rc = pqi_get_logical_device_info(ctrl_info, device);  in pqi_get_device_info_phys_logical()
1756 		rc = pqi_get_physical_device_info(ctrl_info, device, id_phys);  in pqi_get_device_info_phys_logical()
1761 static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,  in pqi_get_device_info()  argument
1767 	rc = pqi_get_device_info_phys_logical(ctrl_info, device, id_phys);  in pqi_get_device_info()
1775 static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,  in pqi_show_volume_status()  argument
1872 	dev_info(&ctrl_info->pci_dev->dev,  in pqi_show_volume_status()
1874 		ctrl_info->scsi_host->host_no,  in pqi_show_volume_status()
1880 	struct pqi_ctrl_info *ctrl_info;  in pqi_rescan_worker()  local
1882 	ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,  in pqi_rescan_worker()
1885 	pqi_scan_scsi_devices(ctrl_info);  in pqi_rescan_worker()
1888 static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,  in pqi_add_device()  argument
1894 		rc = scsi_add_device(ctrl_info->scsi_host, device->bus,  in pqi_add_device()
1897 		rc = pqi_add_sas_device(ctrl_info->sas_host, device);  in pqi_add_device()
1904 static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device)  in pqi_remove_device()  argument
1910 		rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun,  in pqi_remove_device()
1913 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_remove_device()
1915 				ctrl_info->scsi_host->host_no, device->bus,  in pqi_remove_device()
1930 static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,  in pqi_find_scsi_dev()  argument
1935 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)  in pqi_find_scsi_dev()
1959 static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_find_entry()  argument
1964 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {  in pqi_scsi_find_entry()
1989 static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,  in pqi_dev_info()  argument
1996 		"%d:%d:", ctrl_info->scsi_host->host_no, device->bus);  in pqi_dev_info()
2047 	dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);  in pqi_dev_info()
2069 static void pqi_scsi_update_device(struct pqi_ctrl_info *ctrl_info,  in pqi_scsi_update_device()  argument
2096 			if (ctrl_info->logical_volume_rescan_needed)  in pqi_scsi_update_device()
2137 static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,  in pqi_fixup_botched_add()  argument
2142 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
2144 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_fixup_botched_add()
2167 static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,  in pqi_update_device_list()  argument
2187 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
2190 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)  in pqi_update_device_list()
2196 		find_result = pqi_scsi_find_entry(ctrl_info, device,  in pqi_update_device_list()
2207 			pqi_scsi_update_device(ctrl_info, matching_device, device);  in pqi_update_device_list()
2227 	list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,  in pqi_update_device_list()
2243 			&ctrl_info->scsi_device_list);  in pqi_update_device_list()
2250 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_update_device_list()
2257 	if (pqi_ofa_in_progress(ctrl_info)) {  in pqi_update_device_list()
2261 		pqi_ctrl_unblock_device_reset(ctrl_info);  in pqi_update_device_list()
2262 		pqi_scsi_unblock_requests(ctrl_info);  in pqi_update_device_list()
2268 			pqi_dev_info(ctrl_info, "offline", device);  in pqi_update_device_list()
2269 			pqi_show_volume_status(ctrl_info, device);  in pqi_update_device_list()
2271 			pqi_dev_info(ctrl_info, "removed", device);  in pqi_update_device_list()
2274 			pqi_remove_device(ctrl_info, device);  in pqi_update_device_list()
2283 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry) {  in pqi_update_device_list()
2297 			rc = pqi_add_device(ctrl_info, device);  in pqi_update_device_list()
2299 				pqi_dev_info(ctrl_info, "added", device);  in pqi_update_device_list()
2301 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_device_list()
2303 					ctrl_info->scsi_host->host_no,  in pqi_update_device_list()
2306 				pqi_fixup_botched_add(ctrl_info, device);  in pqi_update_device_list()
2311 	ctrl_info->logical_volume_rescan_needed = false;  in pqi_update_device_list()
2357 static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_update_scsi_devices()  argument
2381 	rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);  in pqi_update_scsi_devices()
2408 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2427 		ctrl_info->lv_drive_type_mix_valid = true;  in pqi_update_scsi_devices()
2435 		dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);  in pqi_update_scsi_devices()
2443 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2496 		rc = pqi_get_device_info(ctrl_info, device, id_phys);  in pqi_update_scsi_devices()
2498 			dev_warn(&ctrl_info->pci_dev->dev, "%s\n",  in pqi_update_scsi_devices()
2504 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2509 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_update_scsi_devices()
2542 	pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);  in pqi_update_scsi_devices()
2561 static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)  in pqi_scan_scsi_devices()  argument
2566 	if (pqi_ctrl_offline(ctrl_info))  in pqi_scan_scsi_devices()
2569 	mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
2572 		if (pqi_ctrl_scan_blocked(ctrl_info))  in pqi_scan_scsi_devices()
2574 		pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2578 	rc = pqi_update_scsi_devices(ctrl_info);  in pqi_scan_scsi_devices()
2579 	if (rc && !pqi_ctrl_scan_blocked(ctrl_info))  in pqi_scan_scsi_devices()
2580 		pqi_schedule_rescan_worker_delayed(ctrl_info);  in pqi_scan_scsi_devices()
2582 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_scan_scsi_devices()
2589 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_start()  local
2591 	ctrl_info = shost_to_hba(shost);  in pqi_scan_start()
2593 	pqi_scan_scsi_devices(ctrl_info);  in pqi_scan_start()
2601 	struct pqi_ctrl_info *ctrl_info;  in pqi_scan_finished()  local
2603 	ctrl_info = shost_priv(shost);  in pqi_scan_finished()
2605 	return !mutex_is_locked(&ctrl_info->scan_mutex);  in pqi_scan_finished()
2632 static bool pqi_aio_raid_level_supported(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_raid_level_supported()  argument
2641 		if (rmd->is_write && (!ctrl_info->enable_r1_writes ||  in pqi_aio_raid_level_supported()
2642 			rmd->data_length > ctrl_info->max_write_raid_1_10_2drive))  in pqi_aio_raid_level_supported()
2646 		if (rmd->is_write && (!ctrl_info->enable_r1_writes ||  in pqi_aio_raid_level_supported()
2647 			rmd->data_length > ctrl_info->max_write_raid_1_10_3drive))  in pqi_aio_raid_level_supported()
2651 		if (rmd->is_write && (!ctrl_info->enable_r5_writes ||  in pqi_aio_raid_level_supported()
2652 			rmd->data_length > ctrl_info->max_write_raid_5_6))  in pqi_aio_raid_level_supported()
2656 		if (rmd->is_write && (!ctrl_info->enable_r6_writes ||  in pqi_aio_raid_level_supported()
2657 			rmd->data_length > ctrl_info->max_write_raid_5_6))  in pqi_aio_raid_level_supported()
2716 static int pci_get_aio_common_raid_map_values(struct pqi_ctrl_info *ctrl_info,  in pci_get_aio_common_raid_map_values()  argument
2950 static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_bypass_submit_scsi_cmd()  argument
2968 	if (!pqi_aio_raid_level_supported(ctrl_info, &rmd))  in pqi_raid_bypass_submit_scsi_cmd()
2976 	rc = pci_get_aio_common_raid_map_values(ctrl_info, &rmd, raid_map);  in pqi_raid_bypass_submit_scsi_cmd()
3033 			return pqi_aio_submit_r1_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3037 			return pqi_aio_submit_r56_write_io(ctrl_info, scmd, queue_group,  in pqi_raid_bypass_submit_scsi_cmd()
3042 	return pqi_aio_submit_io(ctrl_info, scmd, rmd.aio_handle,  in pqi_raid_bypass_submit_scsi_cmd()
3061 static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_mode_ready()  argument
3068 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_mode_ready()
3077 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
3089 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
3101 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_wait_for_pqi_mode_ready()
3122 	struct pqi_ctrl_info *ctrl_info;  in pqi_take_device_offline()  local
3130 	ctrl_info = shost_to_hba(sdev->host);  in pqi_take_device_offline()
3131 	pqi_schedule_rescan_worker(ctrl_info);  in pqi_take_device_offline()
3132 	dev_err(&ctrl_info->pci_dev->dev, "re-scanning %s scsi %d:%d:%d:%d\n",  in pqi_take_device_offline()
3133 		path, ctrl_info->scsi_host->host_no, device->bus,  in pqi_take_device_offline()
3208 			struct pqi_ctrl_info *ctrl_info = shost_to_hba(scmd->device->host);  in pqi_process_raid_io_error()  local
3215 						ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
3223 						sshdr.ascq, ctrl_info->scsi_host->host_no, device->bus, device->target, device->lun);  in pqi_process_raid_io_error()
3348 static int pqi_interpret_task_management_response(struct pqi_ctrl_info *ctrl_info,  in pqi_interpret_task_management_response()  argument
3370 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_interpret_task_management_response()
3376 static inline void pqi_invalid_response(struct pqi_ctrl_info *ctrl_info,  in pqi_invalid_response()  argument
3379 	pqi_take_ctrl_offline(ctrl_info, ctrl_shutdown_reason);  in pqi_invalid_response()
3382 static int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info, struct pqi_queue_group *queue_group)  in pqi_process_io_intr()  argument
3396 		if (oq_pi >= ctrl_info->num_elements_per_oq) {  in pqi_process_io_intr()
3397 			pqi_invalid_response(ctrl_info, PQI_IO_PI_OUT_OF_RANGE);  in pqi_process_io_intr()
3398 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
3400 				oq_pi, ctrl_info->num_elements_per_oq - 1, oq_ci);  in pqi_process_io_intr()
3411 		if (request_id >= ctrl_info->max_io_slots) {  in pqi_process_io_intr()
3412 			pqi_invalid_response(ctrl_info, PQI_INVALID_REQ_ID);  in pqi_process_io_intr()
3413 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
3415 				request_id, ctrl_info->max_io_slots - 1, oq_pi, oq_ci);  in pqi_process_io_intr()
3419 		io_request = &ctrl_info->io_request_pool[request_id];  in pqi_process_io_intr()
3421 			pqi_invalid_response(ctrl_info, PQI_UNMATCHED_REQ_ID);  in pqi_process_io_intr()
3422 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
3442 			io_request->status = pqi_interpret_task_management_response(ctrl_info,  in pqi_process_io_intr()
3451 			io_request->error_info = ctrl_info->error_buffer +  in pqi_process_io_intr()
3457 			pqi_invalid_response(ctrl_info, PQI_UNEXPECTED_IU_TYPE);  in pqi_process_io_intr()
3458 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_io_intr()
3470 		oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;  in pqi_process_io_intr()
3494 static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,  in pqi_send_event_ack()  argument
3503 	queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];  in pqi_send_event_ack()
3513 			ctrl_info->num_elements_per_iq))  in pqi_send_event_ack()
3519 		if (pqi_ctrl_offline(ctrl_info))  in pqi_send_event_ack()
3528 	iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;  in pqi_send_event_ack()
3540 static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,  in pqi_acknowledge_event()  argument
3554 	pqi_send_event_ack(ctrl_info, &request, sizeof(request));  in pqi_acknowledge_event()
3561 	struct pqi_ctrl_info *ctrl_info)  in pqi_poll_for_soft_reset_status()  argument
3569 		status = pqi_read_soft_reset_status(ctrl_info);  in pqi_poll_for_soft_reset_status()
3576 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_soft_reset_status()
3580 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_poll_for_soft_reset_status()
3589 static void pqi_process_soft_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_process_soft_reset()  argument
3595 	if (ctrl_info->soft_reset_handshake_supported)  in pqi_process_soft_reset()
3596 		reset_status = pqi_poll_for_soft_reset_status(ctrl_info);  in pqi_process_soft_reset()
3607 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3609 		sis_soft_reset(ctrl_info);  in pqi_process_soft_reset()
3612 		ctrl_info->pqi_mode_enabled = false;  in pqi_process_soft_reset()
3613 		pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_process_soft_reset()
3614 		rc = pqi_ofa_ctrl_restart(ctrl_info, delay_secs);  in pqi_process_soft_reset()
3615 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3616 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_process_soft_reset()
3617 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3622 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3624 		if (ctrl_info->soft_reset_handshake_supported)  in pqi_process_soft_reset()
3625 			pqi_clear_soft_reset_status(ctrl_info);  in pqi_process_soft_reset()
3626 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3627 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_process_soft_reset()
3628 		pqi_ofa_ctrl_unquiesce(ctrl_info);  in pqi_process_soft_reset()
3633 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_soft_reset()
3636 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_process_soft_reset()
3637 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_process_soft_reset()
3638 		pqi_ofa_ctrl_unquiesce(ctrl_info);  in pqi_process_soft_reset()
3639 		pqi_take_ctrl_offline(ctrl_info, PQI_OFA_RESPONSE_TIMEOUT);  in pqi_process_soft_reset()
3646 	struct pqi_ctrl_info *ctrl_info;  in pqi_ofa_memory_alloc_worker()  local
3648 	ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_memory_alloc_work);  in pqi_ofa_memory_alloc_worker()
3650 	pqi_ctrl_ofa_start(ctrl_info);  in pqi_ofa_memory_alloc_worker()
3651 	pqi_ofa_setup_host_buffer(ctrl_info);  in pqi_ofa_memory_alloc_worker()
3652 	pqi_ofa_host_memory_update(ctrl_info);  in pqi_ofa_memory_alloc_worker()
3657 	struct pqi_ctrl_info *ctrl_info;  in pqi_ofa_quiesce_worker()  local
3660 	ctrl_info = container_of(work, struct pqi_ctrl_info, ofa_quiesce_work);  in pqi_ofa_quiesce_worker()
3662 	event = &ctrl_info->events[pqi_event_type_to_event_index(PQI_EVENT_TYPE_OFA)];  in pqi_ofa_quiesce_worker()
3664 	pqi_ofa_ctrl_quiesce(ctrl_info);  in pqi_ofa_quiesce_worker()
3665 	pqi_acknowledge_event(ctrl_info, event);  in pqi_ofa_quiesce_worker()
3666 	pqi_process_soft_reset(ctrl_info);  in pqi_ofa_quiesce_worker()
3669 static bool pqi_ofa_process_event(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_process_event()  argument
3678 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3680 		schedule_work(&ctrl_info->ofa_memory_alloc_work);  in pqi_ofa_process_event()
3683 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3685 		schedule_work(&ctrl_info->ofa_quiesce_work);  in pqi_ofa_process_event()
3689 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3691 			ctrl_info->ofa_cancel_reason);  in pqi_ofa_process_event()
3692 		pqi_ofa_free_host_buffer(ctrl_info);  in pqi_ofa_process_event()
3693 		pqi_ctrl_ofa_done(ctrl_info);  in pqi_ofa_process_event()
3696 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ofa_process_event()
3705 static void pqi_disable_raid_bypass(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_raid_bypass()  argument
3710 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_disable_raid_bypass()
3712 	list_for_each_entry(device, &ctrl_info->scsi_device_list, scsi_device_list_entry)  in pqi_disable_raid_bypass()
3716 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_disable_raid_bypass()
3723 	struct pqi_ctrl_info *ctrl_info;  in pqi_event_worker()  local
3727 	ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);  in pqi_event_worker()
3729 	pqi_ctrl_busy(ctrl_info);  in pqi_event_worker()
3730 	pqi_wait_if_ctrl_blocked(ctrl_info);  in pqi_event_worker()
3731 	if (pqi_ctrl_offline(ctrl_info))  in pqi_event_worker()
3735 	event = ctrl_info->events;  in pqi_event_worker()
3740 				ack_event = pqi_ofa_process_event(ctrl_info, event);  in pqi_event_worker()
3745 					ctrl_info->logical_volume_rescan_needed = true;  in pqi_event_worker()
3747 					pqi_disable_raid_bypass(ctrl_info);  in pqi_event_worker()
3750 				pqi_acknowledge_event(ctrl_info, event);  in pqi_event_worker()
3758 		pqi_schedule_rescan_worker_with_delay(ctrl_info,  in pqi_event_worker()
3762 	pqi_ctrl_unbusy(ctrl_info);  in pqi_event_worker()
3771 	struct pqi_ctrl_info *ctrl_info = from_timer(ctrl_info, t, heartbeat_timer);  in pqi_heartbeat_timer_handler()  local
3773 	pqi_check_ctrl_health(ctrl_info);  in pqi_heartbeat_timer_handler()
3774 	if (pqi_ctrl_offline(ctrl_info))  in pqi_heartbeat_timer_handler()
3777 	num_interrupts = atomic_read(&ctrl_info->num_interrupts);  in pqi_heartbeat_timer_handler()
3778 	heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);  in pqi_heartbeat_timer_handler()
3780 	if (num_interrupts == ctrl_info->previous_num_interrupts) {  in pqi_heartbeat_timer_handler()
3781 		if (heartbeat_count == ctrl_info->previous_heartbeat_count) {  in pqi_heartbeat_timer_handler()
3782 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_heartbeat_timer_handler()
3785 			pqi_take_ctrl_offline(ctrl_info, PQI_NO_HEARTBEAT);  in pqi_heartbeat_timer_handler()
3789 		ctrl_info->previous_num_interrupts = num_interrupts;  in pqi_heartbeat_timer_handler()
3792 	ctrl_info->previous_heartbeat_count = heartbeat_count;  in pqi_heartbeat_timer_handler()
3793 	mod_timer(&ctrl_info->heartbeat_timer,  in pqi_heartbeat_timer_handler()
3797 static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_start_heartbeat_timer()  argument
3799 	if (!ctrl_info->heartbeat_counter)  in pqi_start_heartbeat_timer()
3802 	ctrl_info->previous_num_interrupts =  in pqi_start_heartbeat_timer()
3803 		atomic_read(&ctrl_info->num_interrupts);  in pqi_start_heartbeat_timer()
3804 	ctrl_info->previous_heartbeat_count =  in pqi_start_heartbeat_timer()
3805 		pqi_read_heartbeat_counter(ctrl_info);  in pqi_start_heartbeat_timer()
3807 	ctrl_info->heartbeat_timer.expires =  in pqi_start_heartbeat_timer()
3809 	add_timer(&ctrl_info->heartbeat_timer);  in pqi_start_heartbeat_timer()
3812 static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)  in pqi_stop_heartbeat_timer()  argument
3814 	del_timer_sync(&ctrl_info->heartbeat_timer);  in pqi_stop_heartbeat_timer()
3817 static void pqi_ofa_capture_event_payload(struct pqi_ctrl_info *ctrl_info,  in pqi_ofa_capture_event_payload()  argument
3822 		ctrl_info->ofa_bytes_requested =  in pqi_ofa_capture_event_payload()
3826 		ctrl_info->ofa_cancel_reason =  in pqi_ofa_capture_event_payload()
3832 static int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)  in pqi_process_event_intr()  argument
3842 	event_queue = &ctrl_info->event_queue;  in pqi_process_event_intr()
3849 			pqi_invalid_response(ctrl_info, PQI_EVENT_PI_OUT_OF_RANGE);  in pqi_process_event_intr()
3850 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_event_intr()
3865 			event = &ctrl_info->events[event_index];  in pqi_process_event_intr()
3872 				pqi_ofa_capture_event_payload(ctrl_info, event, response);  in pqi_process_event_intr()
3881 		schedule_work(&ctrl_info->event_work);  in pqi_process_event_intr()
3889 static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info, bool enable_intx)  in pqi_configure_legacy_intx()  argument
3895 	pqi_registers = ctrl_info->pqi_registers;  in pqi_configure_legacy_intx()
3907 static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,  in pqi_change_irq_mode()  argument
3910 	switch (ctrl_info->irq_mode) {  in pqi_change_irq_mode()
3916 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3917 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3926 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3927 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3932 			pqi_configure_legacy_intx(ctrl_info, false);  in pqi_change_irq_mode()
3939 			sis_enable_msix(ctrl_info);  in pqi_change_irq_mode()
3942 			pqi_configure_legacy_intx(ctrl_info, true);  in pqi_change_irq_mode()
3943 			sis_enable_intx(ctrl_info);  in pqi_change_irq_mode()
3951 	ctrl_info->irq_mode = new_mode;  in pqi_change_irq_mode()
3956 static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)  in pqi_is_valid_irq()  argument
3961 	switch (ctrl_info->irq_mode) {  in pqi_is_valid_irq()
3966 		intx_status = readl(&ctrl_info->pqi_registers->legacy_intx_status);  in pqi_is_valid_irq()
3983 	struct pqi_ctrl_info *ctrl_info;  in pqi_irq_handler()  local
3989 	ctrl_info = queue_group->ctrl_info;  in pqi_irq_handler()
3991 	if (!pqi_is_valid_irq(ctrl_info))  in pqi_irq_handler()
3994 	num_io_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);  in pqi_irq_handler()
3998 	if (irq == ctrl_info->event_irq) {  in pqi_irq_handler()
3999 		num_events_handled = pqi_process_event_intr(ctrl_info);  in pqi_irq_handler()
4007 		atomic_inc(&ctrl_info->num_interrupts);  in pqi_irq_handler()
4009 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);  in pqi_irq_handler()
4010 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);  in pqi_irq_handler()
4016 static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_request_irqs()  argument
4018 	struct pci_dev *pci_dev = ctrl_info->pci_dev;  in pqi_request_irqs()
4022 	ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);  in pqi_request_irqs()
4024 	for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {  in pqi_request_irqs()
4026 			DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);  in pqi_request_irqs()
4033 		ctrl_info->num_msix_vectors_initialized++;  in pqi_request_irqs()
4039 static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)  in pqi_free_irqs()  argument
4043 	for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)  in pqi_free_irqs()
4044 		free_irq(pci_irq_vector(ctrl_info->pci_dev, i),  in pqi_free_irqs()
4045 			&ctrl_info->queue_groups[i]);  in pqi_free_irqs()
4047 	ctrl_info->num_msix_vectors_initialized = 0;  in pqi_free_irqs()
4050 static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_msix_interrupts()  argument
4058 	num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,  in pqi_enable_msix_interrupts()
4059 			PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,  in pqi_enable_msix_interrupts()
4062 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_enable_msix_interrupts()
4068 	ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;  in pqi_enable_msix_interrupts()
4069 	ctrl_info->irq_mode = IRQ_MODE_MSIX;  in pqi_enable_msix_interrupts()
4073 static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_disable_msix_interrupts()  argument
4075 	if (ctrl_info->num_msix_vectors_enabled) {  in pqi_disable_msix_interrupts()
4076 		pci_free_irq_vectors(ctrl_info->pci_dev);  in pqi_disable_msix_interrupts()
4077 		ctrl_info->num_msix_vectors_enabled = 0;  in pqi_disable_msix_interrupts()
4081 static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_operational_queues()  argument
4097 		ctrl_info->num_elements_per_iq;  in pqi_alloc_operational_queues()
4100 		ctrl_info->num_elements_per_oq;  in pqi_alloc_operational_queues()
4101 	num_inbound_queues = ctrl_info->num_queue_groups * 2;  in pqi_alloc_operational_queues()
4102 	num_outbound_queues = ctrl_info->num_queue_groups;  in pqi_alloc_operational_queues()
4103 	num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;  in pqi_alloc_operational_queues()
4135 	ctrl_info->queue_memory_base =  in pqi_alloc_operational_queues()
4136 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_operational_queues()
4137 				   &ctrl_info->queue_memory_base_dma_handle,  in pqi_alloc_operational_queues()
4140 	if (!ctrl_info->queue_memory_base)  in pqi_alloc_operational_queues()
4143 	ctrl_info->queue_memory_length = alloc_length;  in pqi_alloc_operational_queues()
4145 	element_array = PTR_ALIGN(ctrl_info->queue_memory_base,  in pqi_alloc_operational_queues()
4148 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
4149 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4152 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4153 				(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4159 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4160 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4166 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
4167 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4170 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4171 			(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4177 	ctrl_info->event_queue.oq_element_array = element_array;  in pqi_alloc_operational_queues()
4178 	ctrl_info->event_queue.oq_element_array_bus_addr =  in pqi_alloc_operational_queues()
4179 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4180 		(element_array - ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4187 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_alloc_operational_queues()
4188 		queue_group = &ctrl_info->queue_groups[i];  in pqi_alloc_operational_queues()
4191 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4193 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4199 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4201 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4207 			ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4209 			(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4215 	ctrl_info->event_queue.oq_pi = next_queue_index;  in pqi_alloc_operational_queues()
4216 	ctrl_info->event_queue.oq_pi_bus_addr =  in pqi_alloc_operational_queues()
4217 		ctrl_info->queue_memory_base_dma_handle +  in pqi_alloc_operational_queues()
4219 		(void __iomem *)ctrl_info->queue_memory_base);  in pqi_alloc_operational_queues()
4224 static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_init_operational_queues()  argument
4234 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
4235 		ctrl_info->queue_groups[i].ctrl_info = ctrl_info;  in pqi_init_operational_queues()
4242 	ctrl_info->event_queue.oq_id = next_oq_id++;  in pqi_init_operational_queues()
4243 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
4244 		ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;  in pqi_init_operational_queues()
4245 		ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;  in pqi_init_operational_queues()
4246 		ctrl_info->queue_groups[i].oq_id = next_oq_id++;  in pqi_init_operational_queues()
4253 	ctrl_info->event_queue.int_msg_num = 0;  in pqi_init_operational_queues()
4254 	for (i = 0; i < ctrl_info->num_queue_groups; i++)  in pqi_init_operational_queues()
4255 		ctrl_info->queue_groups[i].int_msg_num = i;  in pqi_init_operational_queues()
4257 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_init_operational_queues()
4258 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);  in pqi_init_operational_queues()
4259 		spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);  in pqi_init_operational_queues()
4260 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);  in pqi_init_operational_queues()
4261 		INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);  in pqi_init_operational_queues()
4265 static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_admin_queues()  argument
4274 	ctrl_info->admin_queue_memory_base =  in pqi_alloc_admin_queues()
4275 		dma_alloc_coherent(&ctrl_info->pci_dev->dev, alloc_length,  in pqi_alloc_admin_queues()
4276 				   &ctrl_info->admin_queue_memory_base_dma_handle,  in pqi_alloc_admin_queues()
4279 	if (!ctrl_info->admin_queue_memory_base)  in pqi_alloc_admin_queues()
4282 	ctrl_info->admin_queue_memory_length = alloc_length;  in pqi_alloc_admin_queues()
4284 	admin_queues = &ctrl_info->admin_queues;  in pqi_alloc_admin_queues()
4285 	admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,  in pqi_alloc_admin_queues()
4297 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
4299 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
4301 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
4303 		ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
4305 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
4307 		(void __iomem *)ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
4309 		ctrl_info->admin_queue_memory_base_dma_handle +  in pqi_alloc_admin_queues()
4311 		(void __iomem *)ctrl_info->admin_queue_memory_base);  in pqi_alloc_admin_queues()
4319 static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_admin_queues()  argument
4327 	pqi_registers = ctrl_info->pqi_registers;  in pqi_create_admin_queues()
4328 	admin_queues = &ctrl_info->admin_queues;  in pqi_create_admin_queues()
4362 	admin_queues->iq_pi = ctrl_info->iomem_base +  in pqi_create_admin_queues()
4365 	admin_queues->oq_ci = ctrl_info->iomem_base +  in pqi_create_admin_queues()
4372 static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request()  argument
4379 	admin_queues = &ctrl_info->admin_queues;  in pqi_submit_admin_request()
4399 static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,  in pqi_poll_for_admin_response()  argument
4407 	admin_queues = &ctrl_info->admin_queues;  in pqi_poll_for_admin_response()
4417 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_poll_for_admin_response()
4421 		if (!sis_is_firmware_running(ctrl_info))  in pqi_poll_for_admin_response()
4436 static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,  in pqi_start_io()  argument
4475 			ctrl_info->num_elements_per_iq))  in pqi_start_io()
4485 			ctrl_info->num_elements_per_iq - iq_pi;  in pqi_start_io()
4499 			ctrl_info->num_elements_per_iq;  in pqi_start_io()
4518 static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_completion_io()  argument
4530 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_completion_io()
4531 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_completion_io()
4576 static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,  in pqi_submit_raid_request_synchronous()  argument
4586 		if (down_interruptible(&ctrl_info->sync_request_sem))  in pqi_submit_raid_request_synchronous()
4589 		down(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4592 	pqi_ctrl_busy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4598 		pqi_wait_if_ctrl_blocked(ctrl_info);  in pqi_submit_raid_request_synchronous()
4600 	if (pqi_ctrl_offline(ctrl_info)) {  in pqi_submit_raid_request_synchronous()
4605 	io_request = pqi_alloc_io_request(ctrl_info, NULL);  in pqi_submit_raid_request_synchronous()
4621 	pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_submit_raid_request_synchronous()
4624 	pqi_wait_for_completion_io(ctrl_info, &wait);  in pqi_submit_raid_request_synchronous()
4638 	pqi_ctrl_unbusy(ctrl_info);  in pqi_submit_raid_request_synchronous()
4639 	up(&ctrl_info->sync_request_sem);  in pqi_submit_raid_request_synchronous()
4664 	struct pqi_ctrl_info *ctrl_info,  in pqi_submit_admin_request_synchronous()  argument
4670 	pqi_submit_admin_request(ctrl_info, request);  in pqi_submit_admin_request_synchronous()
4672 	rc = pqi_poll_for_admin_response(ctrl_info, response);  in pqi_submit_admin_request_synchronous()
4680 static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_report_device_capability()  argument
4702 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_report_device_capability()
4709 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request, &response);  in pqi_report_device_capability()
4711 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_report_device_capability()
4723 	ctrl_info->max_inbound_queues =  in pqi_report_device_capability()
4725 	ctrl_info->max_elements_per_iq =  in pqi_report_device_capability()
4727 	ctrl_info->max_iq_element_length =  in pqi_report_device_capability()
4730 	ctrl_info->max_outbound_queues =  in pqi_report_device_capability()
4732 	ctrl_info->max_elements_per_oq =  in pqi_report_device_capability()
4734 	ctrl_info->max_oq_element_length =  in pqi_report_device_capability()
4741 	ctrl_info->max_inbound_iu_length_per_firmware =  in pqi_report_device_capability()
4744 	ctrl_info->inbound_spanning_supported =  in pqi_report_device_capability()
4746 	ctrl_info->outbound_spanning_supported =  in pqi_report_device_capability()
4755 static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)  in pqi_validate_device_capability()  argument
4757 	if (ctrl_info->max_iq_element_length <  in pqi_validate_device_capability()
4759 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4761 			ctrl_info->max_iq_element_length,  in pqi_validate_device_capability()
4766 	if (ctrl_info->max_oq_element_length <  in pqi_validate_device_capability()
4768 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4770 			ctrl_info->max_oq_element_length,  in pqi_validate_device_capability()
4775 	if (ctrl_info->max_inbound_iu_length_per_firmware <  in pqi_validate_device_capability()
4777 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4779 			ctrl_info->max_inbound_iu_length_per_firmware,  in pqi_validate_device_capability()
4784 	if (!ctrl_info->inbound_spanning_supported) {  in pqi_validate_device_capability()
4785 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4790 	if (ctrl_info->outbound_spanning_supported) {  in pqi_validate_device_capability()
4791 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_validate_device_capability()
4799 static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)  in pqi_create_event_queue()  argument
4806 	event_queue = &ctrl_info->event_queue;  in pqi_create_event_queue()
4831 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_event_queue()
4836 	event_queue->oq_ci = ctrl_info->iomem_base +  in pqi_create_event_queue()
4844 static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,  in pqi_create_queue_group()  argument
4852 	queue_group = &ctrl_info->queue_groups[group_number];  in pqi_create_queue_group()
4870 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4876 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4879 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4884 	queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4905 	put_unaligned_le16(ctrl_info->num_elements_per_iq,  in pqi_create_queue_group()
4911 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4914 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4919 	queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +  in pqi_create_queue_group()
4939 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4942 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4961 	put_unaligned_le16(ctrl_info->num_elements_per_oq,  in pqi_create_queue_group()
4969 	rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,  in pqi_create_queue_group()
4972 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queue_group()
4977 	queue_group->oq_ci = ctrl_info->iomem_base +  in pqi_create_queue_group()
4985 static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_create_queues()  argument
4990 	rc = pqi_create_event_queue(ctrl_info);  in pqi_create_queues()
4992 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
4997 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_create_queues()
4998 		rc = pqi_create_queue_group(ctrl_info, i);  in pqi_create_queues()
5000 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_create_queues()
5002 				i, ctrl_info->num_queue_groups);  in pqi_create_queues()
5013 static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,  in pqi_configure_events()  argument
5036 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
5043 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_configure_events()
5045 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
5056 				put_unaligned_le16(ctrl_info->event_queue.oq_id,  in pqi_configure_events()
5071 	rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_configure_events()
5078 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_configure_events()
5080 	pqi_pci_unmap(ctrl_info->pci_dev,  in pqi_configure_events()
5090 static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)  in pqi_enable_events()  argument
5092 	return pqi_configure_events(ctrl_info, true);  in pqi_enable_events()
5095 static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_free_all_io_requests()  argument
5102 	if (!ctrl_info->io_request_pool)  in pqi_free_all_io_requests()
5105 	dev = &ctrl_info->pci_dev->dev;  in pqi_free_all_io_requests()
5106 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_free_all_io_requests()
5107 	io_request = ctrl_info->io_request_pool;  in pqi_free_all_io_requests()
5109 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_free_all_io_requests()
5119 	kfree(ctrl_info->io_request_pool);  in pqi_free_all_io_requests()
5120 	ctrl_info->io_request_pool = NULL;  in pqi_free_all_io_requests()
5123 static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_error_buffer()  argument
5125 	ctrl_info->error_buffer = dma_alloc_coherent(&ctrl_info->pci_dev->dev,  in pqi_alloc_error_buffer()
5126 				     ctrl_info->error_buffer_length,  in pqi_alloc_error_buffer()
5127 				     &ctrl_info->error_buffer_dma_handle,  in pqi_alloc_error_buffer()
5129 	if (!ctrl_info->error_buffer)  in pqi_alloc_error_buffer()
5135 static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_alloc_io_resources()  argument
5144 	ctrl_info->io_request_pool = kcalloc(ctrl_info->max_io_slots,  in pqi_alloc_io_resources()
5145 		sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);  in pqi_alloc_io_resources()
5147 	if (!ctrl_info->io_request_pool) {  in pqi_alloc_io_resources()
5148 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
5153 	dev = &ctrl_info->pci_dev->dev;  in pqi_alloc_io_resources()
5154 	sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;  in pqi_alloc_io_resources()
5155 	io_request = ctrl_info->io_request_pool;  in pqi_alloc_io_resources()
5157 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_alloc_io_resources()
5158 		io_request->iu = kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);  in pqi_alloc_io_resources()
5161 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
5171 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_alloc_io_resources()
5185 	pqi_free_all_io_requests(ctrl_info);  in pqi_alloc_io_resources()
5195 static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_io_resources()  argument
5200 	ctrl_info->scsi_ml_can_queue =  in pqi_calculate_io_resources()
5201 		ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;  in pqi_calculate_io_resources()
5202 	ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;  in pqi_calculate_io_resources()
5204 	ctrl_info->error_buffer_length =  in pqi_calculate_io_resources()
5205 		ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;  in pqi_calculate_io_resources()
5208 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
5211 		max_transfer_size = min(ctrl_info->max_transfer_size,  in pqi_calculate_io_resources()
5219 	max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);  in pqi_calculate_io_resources()
5223 	ctrl_info->sg_chain_buffer_length =  in pqi_calculate_io_resources()
5226 	ctrl_info->sg_tablesize = max_sg_entries;  in pqi_calculate_io_resources()
5227 	ctrl_info->max_sectors = max_transfer_size / 512;  in pqi_calculate_io_resources()
5230 static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_calculate_queue_resources()  argument
5242 		max_queue_groups = min(ctrl_info->max_inbound_queues / 2,  in pqi_calculate_queue_resources()
5243 			ctrl_info->max_outbound_queues - 1);  in pqi_calculate_queue_resources()
5247 		num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);  in pqi_calculate_queue_resources()
5251 	ctrl_info->num_queue_groups = num_queue_groups;  in pqi_calculate_queue_resources()
5257 	ctrl_info->max_inbound_iu_length =  in pqi_calculate_queue_resources()
5258 		(ctrl_info->max_inbound_iu_length_per_firmware /  in pqi_calculate_queue_resources()
5263 		(ctrl_info->max_inbound_iu_length /  in pqi_calculate_queue_resources()
5270 		ctrl_info->max_elements_per_iq);  in pqi_calculate_queue_resources()
5274 		ctrl_info->max_elements_per_oq);  in pqi_calculate_queue_resources()
5276 	ctrl_info->num_elements_per_iq = num_elements_per_iq;  in pqi_calculate_queue_resources()
5277 	ctrl_info->num_elements_per_oq = num_elements_per_oq;  in pqi_calculate_queue_resources()
5279 	ctrl_info->max_sg_per_iu =  in pqi_calculate_queue_resources()
5280 		((ctrl_info->max_inbound_iu_length -  in pqi_calculate_queue_resources()
5285 	ctrl_info->max_sg_per_r56_iu =  in pqi_calculate_queue_resources()
5286 		((ctrl_info->max_inbound_iu_length -  in pqi_calculate_queue_resources()
5341 static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_raid_sg_list()  argument
5366 		ctrl_info->max_sg_per_iu, &chained);  in pqi_build_raid_sg_list()
5377 static int pqi_build_aio_r1_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_r1_sg_list()  argument
5403 		ctrl_info->max_sg_per_iu, &chained);  in pqi_build_aio_r1_sg_list()
5415 static int pqi_build_aio_r56_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_r56_sg_list()  argument
5439 			ctrl_info->max_sg_per_r56_iu, &chained);  in pqi_build_aio_r56_sg_list()
5451 static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,  in pqi_build_aio_sg_list()  argument
5477 		ctrl_info->max_sg_per_iu, &chained);  in pqi_build_aio_sg_list()
5500 static int pqi_raid_submit_io(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_submit_io()  argument
5509 	io_request = pqi_alloc_io_request(ctrl_info, scmd);  in pqi_raid_submit_io()
5567 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_raid_submit_io()
5573 	rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);  in pqi_raid_submit_io()
5579 	pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);  in pqi_raid_submit_io()
5584 static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_raid_submit_scsi_cmd()  argument
5592 	return pqi_raid_submit_io(ctrl_info, device, scmd, queue_group, io_high_prio);  in pqi_raid_submit_scsi_cmd()
5599 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_retry_needed()  local
5614 	ctrl_info = shost_to_hba(scmd->device->host);  in pqi_raid_bypass_retry_needed()
5615 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_bypass_retry_needed()
5637 static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_scsi_cmd()  argument
5645 	return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,  in pqi_aio_submit_scsi_cmd()
5650 static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_io()  argument
5660 	io_request = pqi_alloc_io_request(ctrl_info, scmd);  in pqi_aio_submit_io()
5678 	if (!raid_bypass && ctrl_info->multi_lun_device_supported)  in pqi_aio_submit_io()
5699 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_aio_submit_io()
5715 	rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);  in pqi_aio_submit_io()
5721 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_io()
5726 static  int pqi_aio_submit_r1_write_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_r1_write_io()  argument
5735 	io_request = pqi_alloc_io_request(ctrl_info, scmd);  in pqi_aio_submit_r1_write_io()
5776 	rc = pqi_build_aio_r1_sg_list(ctrl_info, r1_request, scmd, io_request);  in pqi_aio_submit_r1_write_io()
5782 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r1_write_io()
5787 static int pqi_aio_submit_r56_write_io(struct pqi_ctrl_info *ctrl_info,  in pqi_aio_submit_r56_write_io()  argument
5796 	io_request = pqi_alloc_io_request(ctrl_info, scmd);  in pqi_aio_submit_r56_write_io()
5843 	rc = pqi_build_aio_r56_sg_list(ctrl_info, r56_request, scmd, io_request);  in pqi_aio_submit_r56_write_io()
5849 	pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);  in pqi_aio_submit_r56_write_io()
5854 static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,  in pqi_get_hw_queue()  argument
5899 static bool pqi_is_parity_write_stream(struct pqi_ctrl_info *ctrl_info,  in pqi_is_parity_write_stream()  argument
5910 	if (!ctrl_info->enable_stream_detection)  in pqi_is_parity_write_stream()
5931 	if ((device->raid_level == SA_RAID_5 && !ctrl_info->enable_r5_writes) ||  in pqi_is_parity_write_stream()
5932 		(device->raid_level == SA_RAID_6 && !ctrl_info->enable_r6_writes))  in pqi_is_parity_write_stream()
5977 	struct pqi_ctrl_info *ctrl_info;  in pqi_scsi_queue_command()  local
5998 	ctrl_info = shost_to_hba(shost);  in pqi_scsi_queue_command()
6000 	if (pqi_ctrl_offline(ctrl_info) || pqi_device_in_remove(device)) {  in pqi_scsi_queue_command()
6006 	if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device, lun)) {  in pqi_scsi_queue_command()
6017 	hw_queue = pqi_get_hw_queue(ctrl_info, scmd);  in pqi_scsi_queue_command()
6018 	queue_group = &ctrl_info->queue_groups[hw_queue];  in pqi_scsi_queue_command()
6024 			!pqi_is_parity_write_stream(ctrl_info, scmd)) {  in pqi_scsi_queue_command()
6025 			rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6032 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6035 			rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6037 			rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd, queue_group);  in pqi_scsi_queue_command()
6049 static unsigned int pqi_queued_io_count(struct pqi_ctrl_info *ctrl_info)  in pqi_queued_io_count()  argument
6060 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_queued_io_count()
6061 		queue_group = &ctrl_info->queue_groups[i];  in pqi_queued_io_count()
6073 static unsigned int pqi_nonempty_inbound_queue_count(struct pqi_ctrl_info *ctrl_info)  in pqi_nonempty_inbound_queue_count()  argument
6084 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_nonempty_inbound_queue_count()
6085 		queue_group = &ctrl_info->queue_groups[i];  in pqi_nonempty_inbound_queue_count()
6099 static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_until_inbound_queues_empty()  argument
6112 		queued_io_count = pqi_queued_io_count(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
6113 		nonempty_inbound_queue_count = pqi_nonempty_inbound_queue_count(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
6116 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_until_inbound_queues_empty()
6117 		if (pqi_ctrl_offline(ctrl_info))  in pqi_wait_until_inbound_queues_empty()
6120 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_wait_until_inbound_queues_empty()
6130 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_wait_until_inbound_queues_empty()
6137 static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,  in pqi_fail_io_queued_for_device()  argument
6149 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_fail_io_queued_for_device()
6150 		queue_group = &ctrl_info->queue_groups[i];  in pqi_fail_io_queued_for_device()
6186 static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,  in pqi_device_wait_for_pending_io()  argument
6198 		if (ctrl_info->ctrl_removal_state != PQI_CTRL_GRACEFUL_REMOVAL) {  in pqi_device_wait_for_pending_io()
6199 			pqi_check_ctrl_health(ctrl_info);  in pqi_device_wait_for_pending_io()
6200 			if (pqi_ctrl_offline(ctrl_info))  in pqi_device_wait_for_pending_io()
6205 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_device_wait_for_pending_io()
6207 				ctrl_info->scsi_host->host_no, device->bus, device->target,  in pqi_device_wait_for_pending_io()
6212 			dev_warn(&ctrl_info->pci_dev->dev,  in pqi_device_wait_for_pending_io()
6214 				ctrl_info->scsi_host->host_no, device->bus, device->target,  in pqi_device_wait_for_pending_io()
6234 static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,  in pqi_wait_for_lun_reset_completion()  argument
6250 		pqi_check_ctrl_health(ctrl_info);  in pqi_wait_for_lun_reset_completion()
6251 		if (pqi_ctrl_offline(ctrl_info)) {  in pqi_wait_for_lun_reset_completion()
6258 		dev_warn(&ctrl_info->pci_dev->dev,  in pqi_wait_for_lun_reset_completion()
6260 			ctrl_info->scsi_host->host_no, device->bus, device->target, lun, wait_secs, cmds_outstanding);  in pqi_wait_for_lun_reset_completion()
6268 static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)  in pqi_lun_reset()  argument
6275 	io_request = pqi_alloc_io_request(ctrl_info, NULL);  in pqi_lun_reset()
6288 	if (!pqi_is_logical_device(device) && ctrl_info->multi_lun_device_supported)  in pqi_lun_reset()
6291 	if (ctrl_info->tmf_iu_timeout_supported)  in pqi_lun_reset()
6294 	pqi_start_io(ctrl_info, &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,  in pqi_lun_reset()
6297 	rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, lun, &wait);  in pqi_lun_reset()
6311 static int pqi_lun_reset_with_retries(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device,…  in pqi_lun_reset_with_retries()  argument
6319 		reset_rc = pqi_lun_reset(ctrl_info, device, lun);  in pqi_lun_reset_with_retries()
6328 	wait_rc = pqi_device_wait_for_pending_io(ctrl_info, device, lun, timeout_msecs);  in pqi_lun_reset_with_retries()
6335 static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u8 lun)  in pqi_device_reset()  argument
6339 	pqi_ctrl_block_requests(ctrl_info);  in pqi_device_reset()
6340 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_device_reset()
6341 	pqi_fail_io_queued_for_device(ctrl_info, device, lun);  in pqi_device_reset()
6342 	rc = pqi_wait_until_inbound_queues_empty(ctrl_info);  in pqi_device_reset()
6344 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_device_reset()
6348 		rc = pqi_lun_reset_with_retries(ctrl_info, device, lun);  in pqi_device_reset()
6354 static int pqi_device_reset_handler(struct pqi_ctrl_info *ctrl_info, struct pqi_scsi_dev *device, u…  in pqi_device_reset_handler()  argument
6358 	mutex_lock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset_handler()
6360 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_device_reset_handler()
6362 		ctrl_info->scsi_host->host_no, device->bus, device->target, lun, scmd, scsi_opcode);  in pqi_device_reset_handler()
6364 	pqi_check_ctrl_health(ctrl_info);  in pqi_device_reset_handler()
6365 	if (pqi_ctrl_offline(ctrl_info))  in pqi_device_reset_handler()
6368 		rc = pqi_device_reset(ctrl_info, device, lun);  in pqi_device_reset_handler()
6370 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_device_reset_handler()
6372 		ctrl_info->scsi_host->host_no, device->bus, device->target, lun,  in pqi_device_reset_handler()
6375 	mutex_unlock(&ctrl_info->lun_reset_mutex);  in pqi_device_reset_handler()
6383 	struct pqi_ctrl_info *ctrl_info;  in pqi_eh_device_reset_handler()  local
6388 	ctrl_info = shost_to_hba(shost);  in pqi_eh_device_reset_handler()
6392 	return pqi_device_reset_handler(ctrl_info, device, (u8)scmd->device->lun, scmd, scsi_opcode);  in pqi_eh_device_reset_handler()
6403 …pqi_device_reset_handler(tmf_work->ctrl_info, tmf_work->device, tmf_work->lun, scmd, tmf_work->scs…  in pqi_tmf_worker()
6409 	struct pqi_ctrl_info *ctrl_info;  in pqi_eh_abort_handler()  local
6415 	ctrl_info = shost_to_hba(shost);  in pqi_eh_abort_handler()
6418 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_abort_handler()
6423 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_abort_handler()
6433 		tmf_work->ctrl_info = ctrl_info;  in pqi_eh_abort_handler()
6442 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_eh_abort_handler()
6455 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_alloc()  local
6459 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_alloc()
6461 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
6466 		device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);  in pqi_slave_alloc()
6477 		device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),  in pqi_slave_alloc()
6498 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_alloc()
6505 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_map_queues()  local
6508 			      ctrl_info->pci_dev, 0);  in pqi_map_queues()
6534 	struct pqi_ctrl_info *ctrl_info;  in pqi_slave_destroy()  local
6539 	ctrl_info = shost_to_hba(sdev->host);  in pqi_slave_destroy()
6541 	mutex_acquired = mutex_trylock(&ctrl_info->scan_mutex);  in pqi_slave_destroy()
6547 		mutex_unlock(&ctrl_info->scan_mutex);  in pqi_slave_destroy()
6553 		mutex_unlock(&ctrl_info->scan_mutex);  in pqi_slave_destroy()
6557 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_destroy()
6559 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_slave_destroy()
6561 	mutex_unlock(&ctrl_info->scan_mutex);  in pqi_slave_destroy()
6563 	pqi_dev_info(ctrl_info, "removed", device);  in pqi_slave_destroy()
6567 static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_getpciinfo_ioctl()  argument
6577 	pci_dev = ctrl_info->pci_dev;  in pqi_getpciinfo_ioctl()
6677 static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)  in pqi_passthru_ioctl()  argument
6688 	if (pqi_ctrl_offline(ctrl_info))  in pqi_passthru_ioctl()
6690 	if (pqi_ofa_in_progress(ctrl_info) && pqi_ctrl_blocked(ctrl_info))  in pqi_passthru_ioctl()
6761 		rc = pqi_map_single(ctrl_info->pci_dev,  in pqi_passthru_ioctl()
6772 	if (ctrl_info->raid_iu_timeout_supported)  in pqi_passthru_ioctl()
6775 	rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,  in pqi_passthru_ioctl()
6779 		pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,  in pqi_passthru_ioctl()
6824 	struct pqi_ctrl_info *ctrl_info;  in pqi_ioctl()  local
6826 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ioctl()
6832 		rc = pqi_scan_scsi_devices(ctrl_info);  in pqi_ioctl()
6835 		rc = pqi_getpciinfo_ioctl(ctrl_info, arg);  in pqi_ioctl()
6841 		rc = pqi_passthru_ioctl(ctrl_info, arg);  in pqi_ioctl()
6855 	struct pqi_ctrl_info *ctrl_info;  in pqi_firmware_version_show()  local
6858 	ctrl_info = shost_to_hba(shost);  in pqi_firmware_version_show()
6860 	return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->firmware_version);  in pqi_firmware_version_show()
6873 	struct pqi_ctrl_info *ctrl_info;  in pqi_serial_number_show()  local
6876 	ctrl_info = shost_to_hba(shost);  in pqi_serial_number_show()
6878 	return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->serial_number);  in pqi_serial_number_show()
6885 	struct pqi_ctrl_info *ctrl_info;  in pqi_model_show()  local
6888 	ctrl_info = shost_to_hba(shost);  in pqi_model_show()
6890 	return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->model);  in pqi_model_show()
6897 	struct pqi_ctrl_info *ctrl_info;  in pqi_vendor_show()  local
6900 	ctrl_info = shost_to_hba(shost);  in pqi_vendor_show()
6902 	return scnprintf(buffer, PAGE_SIZE, "%s\n", ctrl_info->vendor);  in pqi_vendor_show()
6959 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_stream_detection_show()  local
6962 			ctrl_info->enable_stream_detection);  in pqi_host_enable_stream_detection_show()
6969 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_stream_detection_store()  local
6978 	ctrl_info->enable_stream_detection = set_stream_detection;  in pqi_host_enable_stream_detection_store()
6987 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_r5_writes_show()  local
6989 	return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r5_writes);  in pqi_host_enable_r5_writes_show()
6996 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_r5_writes_store()  local
7005 	ctrl_info->enable_r5_writes = set_r5_writes;  in pqi_host_enable_r5_writes_store()
7014 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_r6_writes_show()  local
7016 	return scnprintf(buffer, 10, "%x\n", ctrl_info->enable_r6_writes);  in pqi_host_enable_r6_writes_show()
7023 	struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);  in pqi_host_enable_r6_writes_store()  local
7032 	ctrl_info->enable_r6_writes = set_r6_writes;  in pqi_host_enable_r6_writes_store()
7072 	struct pqi_ctrl_info *ctrl_info;  in pqi_unique_id_show()  local
7079 	ctrl_info = shost_to_hba(sdev->host);  in pqi_unique_id_show()
7081 	if (pqi_ctrl_offline(ctrl_info))  in pqi_unique_id_show()
7084 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
7088 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
7097 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_unique_id_show()
7111 	struct pqi_ctrl_info *ctrl_info;  in pqi_lunid_show()  local
7118 	ctrl_info = shost_to_hba(sdev->host);  in pqi_lunid_show()
7120 	if (pqi_ctrl_offline(ctrl_info))  in pqi_lunid_show()
7123 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
7127 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
7133 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_lunid_show()
7143 	struct pqi_ctrl_info *ctrl_info;  in pqi_path_info_show()  local
7156 	ctrl_info = shost_to_hba(sdev->host);  in pqi_path_info_show()
7158 	if (pqi_ctrl_offline(ctrl_info))  in pqi_path_info_show()
7161 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
7165 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
7182 					ctrl_info->scsi_host->host_no,  in pqi_path_info_show()
7221 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_path_info_show()
7229 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_address_show()  local
7236 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_address_show()
7238 	if (pqi_ctrl_offline(ctrl_info))  in pqi_sas_address_show()
7241 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
7245 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
7251 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_address_show()
7259 	struct pqi_ctrl_info *ctrl_info;  in pqi_ssd_smart_path_enabled_show()  local
7265 	ctrl_info = shost_to_hba(sdev->host);  in pqi_ssd_smart_path_enabled_show()
7267 	if (pqi_ctrl_offline(ctrl_info))  in pqi_ssd_smart_path_enabled_show()
7270 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
7274 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
7282 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_ssd_smart_path_enabled_show()
7290 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_level_show()  local
7297 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_level_show()
7299 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_level_show()
7302 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
7306 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
7315 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_level_show()
7323 	struct pqi_ctrl_info *ctrl_info;  in pqi_raid_bypass_cnt_show()  local
7330 	ctrl_info = shost_to_hba(sdev->host);  in pqi_raid_bypass_cnt_show()
7332 	if (pqi_ctrl_offline(ctrl_info))  in pqi_raid_bypass_cnt_show()
7335 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
7339 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
7345 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_raid_bypass_cnt_show()
7353 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_ncq_prio_enable_show()  local
7360 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_ncq_prio_enable_show()
7362 	if (pqi_ctrl_offline(ctrl_info))  in pqi_sas_ncq_prio_enable_show()
7365 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_show()
7369 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_show()
7375 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_show()
7384 	struct pqi_ctrl_info *ctrl_info;  in pqi_sas_ncq_prio_enable_store()  local
7394 	ctrl_info = shost_to_hba(sdev->host);  in pqi_sas_ncq_prio_enable_store()
7396 	spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_store()
7401 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_store()
7406 		spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_store()
7412 	spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);  in pqi_sas_ncq_prio_enable_store()
7421 	struct pqi_ctrl_info *ctrl_info;  in pqi_numa_node_show()  local
7424 	ctrl_info = shost_to_hba(sdev->host);  in pqi_numa_node_show()
7426 	return scnprintf(buffer, PAGE_SIZE, "%d\n", ctrl_info->numa_node);  in pqi_numa_node_show()
7475 static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_register_scsi()  argument
7480 	shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));  in pqi_register_scsi()
7482 		dev_err(&ctrl_info->pci_dev->dev, "scsi_host_alloc failed\n");  in pqi_register_scsi()
7493 	shost->max_sectors = ctrl_info->max_sectors;  in pqi_register_scsi()
7494 	shost->can_queue = ctrl_info->scsi_ml_can_queue;  in pqi_register_scsi()
7496 	shost->sg_tablesize = ctrl_info->sg_tablesize;  in pqi_register_scsi()
7498 	shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);  in pqi_register_scsi()
7500 	shost->nr_hw_queues = ctrl_info->num_queue_groups;  in pqi_register_scsi()
7502 	shost->hostdata[0] = (unsigned long)ctrl_info;  in pqi_register_scsi()
7504 	rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);  in pqi_register_scsi()
7506 		dev_err(&ctrl_info->pci_dev->dev, "scsi_add_host failed\n");  in pqi_register_scsi()
7510 	rc = pqi_add_sas_host(shost, ctrl_info);  in pqi_register_scsi()
7512 		dev_err(&ctrl_info->pci_dev->dev, "add SAS host failed\n");  in pqi_register_scsi()
7516 	ctrl_info->scsi_host = shost;  in pqi_register_scsi()
7528 static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)  in pqi_unregister_scsi()  argument
7532 	pqi_delete_sas_host(ctrl_info);  in pqi_unregister_scsi()
7534 	shost = ctrl_info->scsi_host;  in pqi_unregister_scsi()
7542 static int pqi_wait_for_pqi_reset_completion(struct pqi_ctrl_info *ctrl_info)  in pqi_wait_for_pqi_reset_completion()  argument
7550 	pqi_registers = ctrl_info->pqi_registers;  in pqi_wait_for_pqi_reset_completion()
7559 		if (!sis_is_firmware_running(ctrl_info)) {  in pqi_wait_for_pqi_reset_completion()
7572 static int pqi_reset(struct pqi_ctrl_info *ctrl_info)  in pqi_reset()  argument
7577 	if (ctrl_info->pqi_reset_quiesce_supported) {  in pqi_reset()
7578 		rc = sis_pqi_reset_quiesce(ctrl_info);  in pqi_reset()
7580 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
7590 	writel(reset_reg.all_bits, &ctrl_info->pqi_registers->device_reset);  in pqi_reset()
7592 	rc = pqi_wait_for_pqi_reset_completion(ctrl_info);  in pqi_reset()
7594 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_reset()
7600 static int pqi_get_ctrl_serial_number(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_serial_number()  argument
7609 	rc = pqi_sense_subsystem_info(ctrl_info, sense_info);  in pqi_get_ctrl_serial_number()
7613 	memcpy(ctrl_info->serial_number, sense_info->ctrl_serial_number,  in pqi_get_ctrl_serial_number()
7615 	ctrl_info->serial_number[sizeof(sense_info->ctrl_serial_number)] = '\0';  in pqi_get_ctrl_serial_number()
7623 static int pqi_get_ctrl_product_details(struct pqi_ctrl_info *ctrl_info)  in pqi_get_ctrl_product_details()  argument
7632 	rc = pqi_identify_controller(ctrl_info, identify);  in pqi_get_ctrl_product_details()
7638 		memcpy(ctrl_info->firmware_version,  in pqi_get_ctrl_product_details()
7642 		memcpy(ctrl_info->firmware_version,  in pqi_get_ctrl_product_details()
7645 		ctrl_info->firmware_version  in pqi_get_ctrl_product_details()
7647 		snprintf(ctrl_info->firmware_version +  in pqi_get_ctrl_product_details()
7648 			strlen(ctrl_info->firmware_version),  in pqi_get_ctrl_product_details()
7649 			sizeof(ctrl_info->firmware_version) -  in pqi_get_ctrl_product_details()
7655 	memcpy(ctrl_info->model, identify->product_id,  in pqi_get_ctrl_product_details()
7657 	ctrl_info->model[sizeof(identify->product_id)] = '\0';  in pqi_get_ctrl_product_details()
7659 	memcpy(ctrl_info->vendor, identify->vendor_id,  in pqi_get_ctrl_product_details()
7661 	ctrl_info->vendor[sizeof(identify->vendor_id)] = '\0';  in pqi_get_ctrl_product_details()
7663 	dev_info(&ctrl_info->pci_dev->dev,  in pqi_get_ctrl_product_details()
7664 		"Firmware version: %s\n", ctrl_info->firmware_version);  in pqi_get_ctrl_product_details()
7673 	struct pqi_ctrl_info *ctrl_info;  member
7726 static int pqi_config_table_update(struct pqi_ctrl_info *ctrl_info,  in pqi_config_table_update()  argument
7743 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_config_table_update()
7746 static int pqi_enable_firmware_features(struct pqi_ctrl_info *ctrl_info,  in pqi_enable_firmware_features()  argument
7773 	return pqi_config_table_update(ctrl_info,  in pqi_enable_firmware_features()
7783 	void (*feature_status)(struct pqi_ctrl_info *ctrl_info,
7787 static void pqi_firmware_feature_status(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_status()  argument
7791 		dev_info(&ctrl_info->pci_dev->dev, "%s not supported by controller\n",  in pqi_firmware_feature_status()
7797 		dev_info(&ctrl_info->pci_dev->dev,  in pqi_firmware_feature_status()
7802 	dev_err(&ctrl_info->pci_dev->dev, "failed to enable %s\n",  in pqi_firmware_feature_status()
7806 static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,  in pqi_ctrl_update_feature_flags()  argument
7811 		ctrl_info->enable_r1_writes = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7814 		ctrl_info->enable_r5_writes = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7817 		ctrl_info->enable_r6_writes = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7820 		ctrl_info->soft_reset_handshake_supported =  in pqi_ctrl_update_feature_flags()
7822 			pqi_read_soft_reset_status(ctrl_info);  in pqi_ctrl_update_feature_flags()
7825 		ctrl_info->raid_iu_timeout_supported = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7828 		ctrl_info->tmf_iu_timeout_supported = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7831 		ctrl_info->firmware_triage_supported = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7832 		pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);  in pqi_ctrl_update_feature_flags()
7835 		ctrl_info->rpl_extended_format_4_5_supported = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7838 		ctrl_info->multi_lun_device_supported = firmware_feature->enabled;  in pqi_ctrl_update_feature_flags()
7842 	pqi_firmware_feature_status(ctrl_info, firmware_feature);  in pqi_ctrl_update_feature_flags()
7845 static inline void pqi_firmware_feature_update(struct pqi_ctrl_info *ctrl_info,  in pqi_firmware_feature_update()  argument
7849 		firmware_feature->feature_status(ctrl_info, firmware_feature);  in pqi_firmware_feature_update()
7951 	struct pqi_ctrl_info *ctrl_info;  in pqi_process_firmware_features()  local
7957 	ctrl_info = section_info->ctrl_info;  in pqi_process_firmware_features()
7968 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
7983 	rc = pqi_enable_firmware_features(ctrl_info, firmware_features,  in pqi_process_firmware_features()
7986 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_firmware_features()
7991 			pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
8005 		pqi_firmware_feature_update(ctrl_info,  in pqi_process_firmware_features()
8034 static void pqi_ctrl_reset_config(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_reset_config()  argument
8036 	ctrl_info->heartbeat_counter = NULL;  in pqi_ctrl_reset_config()
8037 	ctrl_info->soft_reset_status = NULL;  in pqi_ctrl_reset_config()
8038 	ctrl_info->soft_reset_handshake_supported = false;  in pqi_ctrl_reset_config()
8039 	ctrl_info->enable_r1_writes = false;  in pqi_ctrl_reset_config()
8040 	ctrl_info->enable_r5_writes = false;  in pqi_ctrl_reset_config()
8041 	ctrl_info->enable_r6_writes = false;  in pqi_ctrl_reset_config()
8042 	ctrl_info->raid_iu_timeout_supported = false;  in pqi_ctrl_reset_config()
8043 	ctrl_info->tmf_iu_timeout_supported = false;  in pqi_ctrl_reset_config()
8044 	ctrl_info->firmware_triage_supported = false;  in pqi_ctrl_reset_config()
8045 	ctrl_info->rpl_extended_format_4_5_supported = false;  in pqi_ctrl_reset_config()
8046 	ctrl_info->multi_lun_device_supported = false;  in pqi_ctrl_reset_config()
8049 static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)  in pqi_process_config_table()  argument
8060 	table_length = ctrl_info->config_table_length;  in pqi_process_config_table()
8066 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
8075 	table_iomem_addr = ctrl_info->iomem_base + ctrl_info->config_table_offset;  in pqi_process_config_table()
8079 	section_info.ctrl_info = ctrl_info;  in pqi_process_config_table()
8096 				dev_warn(&ctrl_info->pci_dev->dev,  in pqi_process_config_table()
8099 				ctrl_info->heartbeat_counter =  in pqi_process_config_table()
8106 			ctrl_info->soft_reset_status =  in pqi_process_config_table()
8132 static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_revert_to_sis_mode()  argument
8136 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);  in pqi_revert_to_sis_mode()
8137 	rc = pqi_reset(ctrl_info);  in pqi_revert_to_sis_mode()
8140 	rc = sis_reenable_sis_mode(ctrl_info);  in pqi_revert_to_sis_mode()
8142 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_revert_to_sis_mode()
8146 	pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_revert_to_sis_mode()
8156 static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)  in pqi_force_sis_mode()  argument
8158 	if (!sis_is_firmware_running(ctrl_info))  in pqi_force_sis_mode()
8161 	if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)  in pqi_force_sis_mode()
8164 	if (sis_is_kernel_up(ctrl_info)) {  in pqi_force_sis_mode()
8165 		pqi_save_ctrl_mode(ctrl_info, SIS_MODE);  in pqi_force_sis_mode()
8169 	return pqi_revert_to_sis_mode(ctrl_info);  in pqi_force_sis_mode()
8187 static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init()  argument
8193 		if (pqi_is_fw_triage_supported(ctrl_info)) {  in pqi_ctrl_init()
8194 			rc = sis_wait_for_fw_triage_completion(ctrl_info);  in pqi_ctrl_init()
8198 		sis_soft_reset(ctrl_info);  in pqi_ctrl_init()
8201 		rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init()
8210 	rc = sis_wait_for_ctrl_ready(ctrl_info);  in pqi_ctrl_init()
8213 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8225 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init()
8227 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8232 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init()
8234 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8239 	product_id = sis_get_product_id(ctrl_info);  in pqi_ctrl_init()
8240 	ctrl_info->product_id = (u8)product_id;  in pqi_ctrl_init()
8241 	ctrl_info->product_revision = (u8)(product_id >> 8);  in pqi_ctrl_init()
8244 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
8246 				ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
8249 		if (ctrl_info->max_outstanding_requests >  in pqi_ctrl_init()
8251 				ctrl_info->max_outstanding_requests =  in pqi_ctrl_init()
8255 	pqi_calculate_io_resources(ctrl_info);  in pqi_ctrl_init()
8257 	rc = pqi_alloc_error_buffer(ctrl_info);  in pqi_ctrl_init()
8259 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8269 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init()
8271 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8277 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init()
8279 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8285 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init()
8286 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init()
8288 	rc = pqi_alloc_admin_queues(ctrl_info);  in pqi_ctrl_init()
8290 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8295 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init()
8297 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8302 	rc = pqi_report_device_capability(ctrl_info);  in pqi_ctrl_init()
8304 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8309 	rc = pqi_validate_device_capability(ctrl_info);  in pqi_ctrl_init()
8313 	pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
8315 	rc = pqi_enable_msix_interrupts(ctrl_info);  in pqi_ctrl_init()
8319 	if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {  in pqi_ctrl_init()
8320 		ctrl_info->max_msix_vectors =  in pqi_ctrl_init()
8321 			ctrl_info->num_msix_vectors_enabled;  in pqi_ctrl_init()
8322 		pqi_calculate_queue_resources(ctrl_info);  in pqi_ctrl_init()
8325 	rc = pqi_alloc_io_resources(ctrl_info);  in pqi_ctrl_init()
8329 	rc = pqi_alloc_operational_queues(ctrl_info);  in pqi_ctrl_init()
8331 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8336 	pqi_init_operational_queues(ctrl_info);  in pqi_ctrl_init()
8338 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init()
8342 	rc = pqi_request_irqs(ctrl_info);  in pqi_ctrl_init()
8346 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init()
8348 	ctrl_info->controller_online = true;  in pqi_ctrl_init()
8350 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init()
8354 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init()
8356 	if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {  in pqi_ctrl_init()
8357 		rc = pqi_get_advanced_raid_bypass_config(ctrl_info);  in pqi_ctrl_init()
8359 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8363 		ctrl_info->ciss_report_log_flags |=  in pqi_ctrl_init()
8367 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init()
8369 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8375 	rc = pqi_register_scsi(ctrl_info);  in pqi_ctrl_init()
8379 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init()
8381 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8386 	rc = pqi_get_ctrl_serial_number(ctrl_info);  in pqi_ctrl_init()
8388 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8393 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init()
8395 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8400 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init()
8402 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init()
8407 	pqi_schedule_update_time_worker(ctrl_info);  in pqi_ctrl_init()
8409 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init()
8414 static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)  in pqi_reinit_queues()  argument
8420 	admin_queues = &ctrl_info->admin_queues;  in pqi_reinit_queues()
8425 	for (i = 0; i < ctrl_info->num_queue_groups; i++) {  in pqi_reinit_queues()
8426 		ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;  in pqi_reinit_queues()
8427 		ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;  in pqi_reinit_queues()
8428 		ctrl_info->queue_groups[i].oq_ci_copy = 0;  in pqi_reinit_queues()
8430 		writel(0, ctrl_info->queue_groups[i].iq_ci[RAID_PATH]);  in pqi_reinit_queues()
8431 		writel(0, ctrl_info->queue_groups[i].iq_ci[AIO_PATH]);  in pqi_reinit_queues()
8432 		writel(0, ctrl_info->queue_groups[i].oq_pi);  in pqi_reinit_queues()
8435 	event_queue = &ctrl_info->event_queue;  in pqi_reinit_queues()
8440 static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)  in pqi_ctrl_init_resume()  argument
8444 	rc = pqi_force_sis_mode(ctrl_info);  in pqi_ctrl_init_resume()
8452 	rc = sis_wait_for_ctrl_ready_resume(ctrl_info);  in pqi_ctrl_init_resume()
8460 	rc = sis_get_ctrl_properties(ctrl_info);  in pqi_ctrl_init_resume()
8462 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8467 	rc = sis_get_pqi_capabilities(ctrl_info);  in pqi_ctrl_init_resume()
8469 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8479 	rc = sis_init_base_struct_addr(ctrl_info);  in pqi_ctrl_init_resume()
8481 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8487 	rc = pqi_wait_for_pqi_mode_ready(ctrl_info);  in pqi_ctrl_init_resume()
8489 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8495 	ctrl_info->pqi_mode_enabled = true;  in pqi_ctrl_init_resume()
8496 	pqi_save_ctrl_mode(ctrl_info, PQI_MODE);  in pqi_ctrl_init_resume()
8498 	pqi_reinit_queues(ctrl_info);  in pqi_ctrl_init_resume()
8500 	rc = pqi_create_admin_queues(ctrl_info);  in pqi_ctrl_init_resume()
8502 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8507 	rc = pqi_create_queues(ctrl_info);  in pqi_ctrl_init_resume()
8511 	pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);  in pqi_ctrl_init_resume()
8513 	ctrl_info->controller_online = true;  in pqi_ctrl_init_resume()
8514 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ctrl_init_resume()
8516 	pqi_ctrl_reset_config(ctrl_info);  in pqi_ctrl_init_resume()
8518 	rc = pqi_process_config_table(ctrl_info);  in pqi_ctrl_init_resume()
8522 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ctrl_init_resume()
8524 	if (ctrl_info->enable_r5_writes || ctrl_info->enable_r6_writes) {  in pqi_ctrl_init_resume()
8525 		rc = pqi_get_advanced_raid_bypass_config(ctrl_info);  in pqi_ctrl_init_resume()
8527 			dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8531 		ctrl_info->ciss_report_log_flags |=  in pqi_ctrl_init_resume()
8535 	rc = pqi_enable_events(ctrl_info);  in pqi_ctrl_init_resume()
8537 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8542 	rc = pqi_get_ctrl_product_details(ctrl_info);  in pqi_ctrl_init_resume()
8544 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8549 	rc = pqi_set_diag_rescan(ctrl_info);  in pqi_ctrl_init_resume()
8551 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8556 	rc = pqi_write_driver_version_to_host_wellness(ctrl_info);  in pqi_ctrl_init_resume()
8558 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_ctrl_init_resume()
8563 	if (pqi_ofa_in_progress(ctrl_info))  in pqi_ctrl_init_resume()
8564 		pqi_ctrl_unblock_scan(ctrl_info);  in pqi_ctrl_init_resume()
8566 	pqi_scan_scsi_devices(ctrl_info);  in pqi_ctrl_init_resume()
8581 static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_pci_init()  argument
8586 	rc = pci_enable_device(ctrl_info->pci_dev);  in pqi_pci_init()
8588 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
8598 	rc = dma_set_mask_and_coherent(&ctrl_info->pci_dev->dev, mask);  in pqi_pci_init()
8600 		dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");  in pqi_pci_init()
8604 	rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);  in pqi_pci_init()
8606 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
8611 	ctrl_info->iomem_base = ioremap(pci_resource_start(  in pqi_pci_init()
8612 		ctrl_info->pci_dev, 0),  in pqi_pci_init()
8613 		pci_resource_len(ctrl_info->pci_dev, 0));  in pqi_pci_init()
8614 	if (!ctrl_info->iomem_base) {  in pqi_pci_init()
8615 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
8624 	rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,  in pqi_pci_init()
8627 		dev_err(&ctrl_info->pci_dev->dev,  in pqi_pci_init()
8633 	pci_set_master(ctrl_info->pci_dev);  in pqi_pci_init()
8635 	ctrl_info->registers = ctrl_info->iomem_base;  in pqi_pci_init()
8636 	ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;  in pqi_pci_init()
8638 	pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);  in pqi_pci_init()
8643 	pci_release_regions(ctrl_info->pci_dev);  in pqi_pci_init()
8645 	pci_disable_device(ctrl_info->pci_dev);  in pqi_pci_init()
8650 static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)  in pqi_cleanup_pci_init()  argument
8652 	iounmap(ctrl_info->iomem_base);  in pqi_cleanup_pci_init()
8653 	pci_release_regions(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
8654 	if (pci_is_enabled(ctrl_info->pci_dev))  in pqi_cleanup_pci_init()
8655 		pci_disable_device(ctrl_info->pci_dev);  in pqi_cleanup_pci_init()
8656 	pci_set_drvdata(ctrl_info->pci_dev, NULL);  in pqi_cleanup_pci_init()
8661 	struct pqi_ctrl_info *ctrl_info;  in pqi_alloc_ctrl_info()  local
8663 	ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),  in pqi_alloc_ctrl_info()
8665 	if (!ctrl_info)  in pqi_alloc_ctrl_info()
8668 	mutex_init(&ctrl_info->scan_mutex);  in pqi_alloc_ctrl_info()
8669 	mutex_init(&ctrl_info->lun_reset_mutex);  in pqi_alloc_ctrl_info()
8670 	mutex_init(&ctrl_info->ofa_mutex);  in pqi_alloc_ctrl_info()
8672 	INIT_LIST_HEAD(&ctrl_info->scsi_device_list);  in pqi_alloc_ctrl_info()
8673 	spin_lock_init(&ctrl_info->scsi_device_list_lock);  in pqi_alloc_ctrl_info()
8675 	INIT_WORK(&ctrl_info->event_work, pqi_event_worker);  in pqi_alloc_ctrl_info()
8676 	atomic_set(&ctrl_info->num_interrupts, 0);  in pqi_alloc_ctrl_info()
8678 	INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);  in pqi_alloc_ctrl_info()
8679 	INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);  in pqi_alloc_ctrl_info()
8681 	timer_setup(&ctrl_info->heartbeat_timer, pqi_heartbeat_timer_handler, 0);  in pqi_alloc_ctrl_info()
8682 	INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);  in pqi_alloc_ctrl_info()
8684 	INIT_WORK(&ctrl_info->ofa_memory_alloc_work, pqi_ofa_memory_alloc_worker);  in pqi_alloc_ctrl_info()
8685 	INIT_WORK(&ctrl_info->ofa_quiesce_work, pqi_ofa_quiesce_worker);  in pqi_alloc_ctrl_info()
8687 	sema_init(&ctrl_info->sync_request_sem,  in pqi_alloc_ctrl_info()
8689 	init_waitqueue_head(&ctrl_info->block_requests_wait);  in pqi_alloc_ctrl_info()
8691 	ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;  in pqi_alloc_ctrl_info()
8692 	ctrl_info->irq_mode = IRQ_MODE_NONE;  in pqi_alloc_ctrl_info()
8693 	ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;  in pqi_alloc_ctrl_info()
8695 	ctrl_info->ciss_report_log_flags = CISS_REPORT_LOG_FLAG_UNIQUE_LUN_ID;  in pqi_alloc_ctrl_info()
8696 	ctrl_info->max_transfer_encrypted_sas_sata =  in pqi_alloc_ctrl_info()
8698 	ctrl_info->max_transfer_encrypted_nvme =  in pqi_alloc_ctrl_info()
8700 	ctrl_info->max_write_raid_5_6 = PQI_DEFAULT_MAX_WRITE_RAID_5_6;  in pqi_alloc_ctrl_info()
8701 	ctrl_info->max_write_raid_1_10_2drive = ~0;  in pqi_alloc_ctrl_info()
8702 	ctrl_info->max_write_raid_1_10_3drive = ~0;  in pqi_alloc_ctrl_info()
8703 	ctrl_info->disable_managed_interrupts = pqi_disable_managed_interrupts;  in pqi_alloc_ctrl_info()
8705 	return ctrl_info;  in pqi_alloc_ctrl_info()
8708 static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_info()  argument
8710 	kfree(ctrl_info);  in pqi_free_ctrl_info()
8713 static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)  in pqi_free_interrupts()  argument
8715 	pqi_free_irqs(ctrl_info);  in pqi_free_interrupts()
8716 	pqi_disable_msix_interrupts(ctrl_info);  in pqi_free_interrupts()
8719 static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)  in pqi_free_ctrl_resources()  argument
8721 	pqi_free_interrupts(ctrl_info);  in pqi_free_ctrl_resources()
8722 	if (ctrl_info->queue_memory_base)  in pqi_free_ctrl_resources()
8723 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
8724 			ctrl_info->queue_memory_length,  in pqi_free_ctrl_resources()
8725 			ctrl_info->queue_memory_base,  in pqi_free_ctrl_resources()
8726 			ctrl_info->queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
8727 	if (ctrl_info->admin_queue_memory_base)  in pqi_free_ctrl_resources()
8728 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
8729 			ctrl_info->admin_queue_memory_length,  in pqi_free_ctrl_resources()
8730 			ctrl_info->admin_queue_memory_base,  in pqi_free_ctrl_resources()
8731 			ctrl_info->admin_queue_memory_base_dma_handle);  in pqi_free_ctrl_resources()
8732 	pqi_free_all_io_requests(ctrl_info);  in pqi_free_ctrl_resources()
8733 	if (ctrl_info->error_buffer)  in pqi_free_ctrl_resources()
8734 		dma_free_coherent(&ctrl_info->pci_dev->dev,  in pqi_free_ctrl_resources()
8735 			ctrl_info->error_buffer_length,  in pqi_free_ctrl_resources()
8736 			ctrl_info->error_buffer,  in pqi_free_ctrl_resources()
8737 			ctrl_info->error_buffer_dma_handle);  in pqi_free_ctrl_resources()
8738 	if (ctrl_info->iomem_base)  in pqi_free_ctrl_resources()
8739 		pqi_cleanup_pci_init(ctrl_info);  in pqi_free_ctrl_resources()
8740 	pqi_free_ctrl_info(ctrl_info);  in pqi_free_ctrl_resources()
8743 static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)  in pqi_remove_ctrl()  argument
8745 	ctrl_info->controller_online = false;  in pqi_remove_ctrl()
8746 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_remove_ctrl()
8747 	pqi_ctrl_block_requests(ctrl_info);  in pqi_remove_ctrl()
8748 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_remove_ctrl()
8749 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_remove_ctrl()
8750 	if (ctrl_info->ctrl_removal_state == PQI_CTRL_SURPRISE_REMOVAL) {  in pqi_remove_ctrl()
8751 		pqi_fail_all_outstanding_requests(ctrl_info);  in pqi_remove_ctrl()
8752 		ctrl_info->pqi_mode_enabled = false;  in pqi_remove_ctrl()
8754 	pqi_unregister_scsi(ctrl_info);  in pqi_remove_ctrl()
8755 	if (ctrl_info->pqi_mode_enabled)  in pqi_remove_ctrl()
8756 		pqi_revert_to_sis_mode(ctrl_info);  in pqi_remove_ctrl()
8757 	pqi_free_ctrl_resources(ctrl_info);  in pqi_remove_ctrl()
8760 static void pqi_ofa_ctrl_quiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_quiesce()  argument
8762 	pqi_ctrl_block_scan(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8763 	pqi_scsi_block_requests(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8764 	pqi_ctrl_block_device_reset(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8765 	pqi_ctrl_block_requests(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8766 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8767 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_quiesce()
8770 static void pqi_ofa_ctrl_unquiesce(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_ctrl_unquiesce()  argument
8772 	pqi_start_heartbeat_timer(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
8773 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
8774 	pqi_ctrl_unblock_device_reset(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
8775 	pqi_scsi_unblock_requests(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
8776 	pqi_ctrl_unblock_scan(ctrl_info);  in pqi_ofa_ctrl_unquiesce()
8779 static int pqi_ofa_alloc_mem(struct pqi_ctrl_info *ctrl_info, u32 total_size, u32 chunk_size)  in pqi_ofa_alloc_mem()  argument
8788 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_alloc_mem()
8794 	ctrl_info->pqi_ofa_chunk_virt_addr = kmalloc_array(sg_count, sizeof(void *), GFP_KERNEL);  in pqi_ofa_alloc_mem()
8795 	if (!ctrl_info->pqi_ofa_chunk_virt_addr)  in pqi_ofa_alloc_mem()
8798 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_alloc_mem()
8801 		ctrl_info->pqi_ofa_chunk_virt_addr[i] =  in pqi_ofa_alloc_mem()
8803 		if (!ctrl_info->pqi_ofa_chunk_virt_addr[i])  in pqi_ofa_alloc_mem()
8820 			ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_alloc_mem()
8823 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_alloc_mem()
8829 static int pqi_ofa_alloc_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_alloc_host_buffer()  argument
8835 	if (ctrl_info->ofa_bytes_requested == 0)  in pqi_ofa_alloc_host_buffer()
8838 	total_size = PAGE_ALIGN(ctrl_info->ofa_bytes_requested);  in pqi_ofa_alloc_host_buffer()
8843 		if (pqi_ofa_alloc_mem(ctrl_info, total_size, chunk_size) == 0)  in pqi_ofa_alloc_host_buffer()
8852 static void pqi_ofa_setup_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_setup_host_buffer()  argument
8857 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_setup_host_buffer()
8860 		&ctrl_info->pqi_ofa_mem_dma_handle, GFP_KERNEL);  in pqi_ofa_setup_host_buffer()
8864 	ctrl_info->pqi_ofa_mem_virt_addr = ofap;  in pqi_ofa_setup_host_buffer()
8866 	if (pqi_ofa_alloc_host_buffer(ctrl_info) < 0) {  in pqi_ofa_setup_host_buffer()
8869 		dma_free_coherent(dev, sizeof(*ofap), ofap, ctrl_info->pqi_ofa_mem_dma_handle);  in pqi_ofa_setup_host_buffer()
8870 		ctrl_info->pqi_ofa_mem_virt_addr = NULL;  in pqi_ofa_setup_host_buffer()
8878 static void pqi_ofa_free_host_buffer(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_free_host_buffer()  argument
8886 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_free_host_buffer()
8890 	dev = &ctrl_info->pci_dev->dev;  in pqi_ofa_free_host_buffer()
8902 			ctrl_info->pqi_ofa_chunk_virt_addr[i],  in pqi_ofa_free_host_buffer()
8905 	kfree(ctrl_info->pqi_ofa_chunk_virt_addr);  in pqi_ofa_free_host_buffer()
8909 		ctrl_info->pqi_ofa_mem_dma_handle);  in pqi_ofa_free_host_buffer()
8910 	ctrl_info->pqi_ofa_mem_virt_addr = NULL;  in pqi_ofa_free_host_buffer()
8913 static int pqi_ofa_host_memory_update(struct pqi_ctrl_info *ctrl_info)  in pqi_ofa_host_memory_update()  argument
8927 	ofap = ctrl_info->pqi_ofa_mem_virt_addr;  in pqi_ofa_host_memory_update()
8934 		put_unaligned_le64((u64)ctrl_info->pqi_ofa_mem_dma_handle,  in pqi_ofa_host_memory_update()
8940 	return pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0, NULL);  in pqi_ofa_host_memory_update()
8943 static int pqi_ofa_ctrl_restart(struct pqi_ctrl_info *ctrl_info, unsigned int delay_secs)  in pqi_ofa_ctrl_restart()  argument
8947 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_ofa_ctrl_restart()
8955 static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)  in pqi_fail_all_outstanding_requests()  argument
8962 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_fail_all_outstanding_requests()
8963 		io_request = &ctrl_info->io_request_pool[i];  in pqi_fail_all_outstanding_requests()
8987 static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)  in pqi_take_ctrl_offline_deferred()  argument
8990 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8991 	pqi_free_interrupts(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8992 	pqi_cancel_rescan_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8993 	pqi_cancel_update_time_worker(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8994 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8995 	pqi_fail_all_outstanding_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
8996 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_take_ctrl_offline_deferred()
9001 	struct pqi_ctrl_info *ctrl_info;  in pqi_ctrl_offline_worker()  local
9003 	ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);  in pqi_ctrl_offline_worker()
9004 	pqi_take_ctrl_offline_deferred(ctrl_info);  in pqi_ctrl_offline_worker()
9053 static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info,  in pqi_take_ctrl_offline()  argument
9056 	if (!ctrl_info->controller_online)  in pqi_take_ctrl_offline()
9059 	ctrl_info->controller_online = false;  in pqi_take_ctrl_offline()
9060 	ctrl_info->pqi_mode_enabled = false;  in pqi_take_ctrl_offline()
9061 	pqi_ctrl_block_requests(ctrl_info);  in pqi_take_ctrl_offline()
9063 		sis_shutdown_ctrl(ctrl_info, ctrl_shutdown_reason);  in pqi_take_ctrl_offline()
9064 	pci_disable_device(ctrl_info->pci_dev);  in pqi_take_ctrl_offline()
9065 	dev_err(&ctrl_info->pci_dev->dev,  in pqi_take_ctrl_offline()
9068 	schedule_work(&ctrl_info->ctrl_offline_work);  in pqi_take_ctrl_offline()
9089 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_probe()  local
9113 	ctrl_info = pqi_alloc_ctrl_info(node);  in pqi_pci_probe()
9114 	if (!ctrl_info) {  in pqi_pci_probe()
9119 	ctrl_info->numa_node = node;  in pqi_pci_probe()
9121 	ctrl_info->pci_dev = pci_dev;  in pqi_pci_probe()
9123 	rc = pqi_pci_init(ctrl_info);  in pqi_pci_probe()
9127 	rc = pqi_ctrl_init(ctrl_info);  in pqi_pci_probe()
9134 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_probe()
9141 	struct pqi_ctrl_info *ctrl_info;  in pqi_pci_remove()  local
9145 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_pci_remove()
9146 	if (!ctrl_info)  in pqi_pci_remove()
9149 	pci_read_config_word(ctrl_info->pci_dev, PCI_SUBSYSTEM_VENDOR_ID, &vendor_id);  in pqi_pci_remove()
9151 		ctrl_info->ctrl_removal_state = PQI_CTRL_SURPRISE_REMOVAL;  in pqi_pci_remove()
9153 		ctrl_info->ctrl_removal_state = PQI_CTRL_GRACEFUL_REMOVAL;  in pqi_pci_remove()
9155 	if (ctrl_info->ctrl_removal_state == PQI_CTRL_GRACEFUL_REMOVAL) {  in pqi_pci_remove()
9156 		rc = pqi_flush_cache(ctrl_info, RESTART);  in pqi_pci_remove()
9162 	pqi_remove_ctrl(ctrl_info);  in pqi_pci_remove()
9165 static void pqi_crash_if_pending_command(struct pqi_ctrl_info *ctrl_info)  in pqi_crash_if_pending_command()  argument
9171 	for (i = 0; i < ctrl_info->max_io_slots; i++) {  in pqi_crash_if_pending_command()
9172 		io_request = &ctrl_info->io_request_pool[i];  in pqi_crash_if_pending_command()
9184 	struct pqi_ctrl_info *ctrl_info;  in pqi_shutdown()  local
9187 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_shutdown()
9188 	if (!ctrl_info) {  in pqi_shutdown()
9194 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_shutdown()
9196 	pqi_scsi_block_requests(ctrl_info);  in pqi_shutdown()
9197 	pqi_ctrl_block_device_reset(ctrl_info);  in pqi_shutdown()
9198 	pqi_ctrl_block_requests(ctrl_info);  in pqi_shutdown()
9199 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_shutdown()
9210 	rc = pqi_flush_cache(ctrl_info, shutdown_event);  in pqi_shutdown()
9215 	pqi_crash_if_pending_command(ctrl_info);  in pqi_shutdown()
9216 	pqi_reset(ctrl_info);  in pqi_shutdown()
9278 	struct pqi_ctrl_info *ctrl_info;  in pqi_suspend_or_freeze()  local
9281 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_suspend_or_freeze()
9283 	pqi_wait_until_ofa_finished(ctrl_info);  in pqi_suspend_or_freeze()
9285 	pqi_ctrl_block_scan(ctrl_info);  in pqi_suspend_or_freeze()
9286 	pqi_scsi_block_requests(ctrl_info);  in pqi_suspend_or_freeze()
9287 	pqi_ctrl_block_device_reset(ctrl_info);  in pqi_suspend_or_freeze()
9288 	pqi_ctrl_block_requests(ctrl_info);  in pqi_suspend_or_freeze()
9289 	pqi_ctrl_wait_until_quiesced(ctrl_info);  in pqi_suspend_or_freeze()
9295 		pqi_flush_cache(ctrl_info, shutdown_event);  in pqi_suspend_or_freeze()
9298 	pqi_stop_heartbeat_timer(ctrl_info);  in pqi_suspend_or_freeze()
9299 	pqi_crash_if_pending_command(ctrl_info);  in pqi_suspend_or_freeze()
9300 	pqi_free_irqs(ctrl_info);  in pqi_suspend_or_freeze()
9302 	ctrl_info->controller_online = false;  in pqi_suspend_or_freeze()
9303 	ctrl_info->pqi_mode_enabled = false;  in pqi_suspend_or_freeze()
9317 	struct pqi_ctrl_info *ctrl_info;  in pqi_resume_or_restore()  local
9320 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_resume_or_restore()
9322 	rc = pqi_request_irqs(ctrl_info);  in pqi_resume_or_restore()
9326 	pqi_ctrl_unblock_device_reset(ctrl_info);  in pqi_resume_or_restore()
9327 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_resume_or_restore()
9328 	pqi_scsi_unblock_requests(ctrl_info);  in pqi_resume_or_restore()
9329 	pqi_ctrl_unblock_scan(ctrl_info);  in pqi_resume_or_restore()
9333 	return pqi_ctrl_init_resume(ctrl_info);  in pqi_resume_or_restore()
9345 	struct pqi_ctrl_info *ctrl_info;  in pqi_thaw()  local
9348 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_thaw()
9350 	rc = pqi_request_irqs(ctrl_info);  in pqi_thaw()
9354 	ctrl_info->controller_online = true;  in pqi_thaw()
9355 	ctrl_info->pqi_mode_enabled = true;  in pqi_thaw()
9357 	pqi_ctrl_unblock_device_reset(ctrl_info);  in pqi_thaw()
9358 	pqi_ctrl_unblock_requests(ctrl_info);  in pqi_thaw()
9359 	pqi_scsi_unblock_requests(ctrl_info);  in pqi_thaw()
9360 	pqi_ctrl_unblock_scan(ctrl_info);  in pqi_thaw()
9368 	struct pqi_ctrl_info *ctrl_info;  in pqi_poweroff()  local
9372 	ctrl_info = pci_get_drvdata(pci_dev);  in pqi_poweroff()
9375 	pqi_flush_cache(ctrl_info, shutdown_event);  in pqi_poweroff()