67 static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
71 static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
73 static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
75 static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
76 static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
79 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
80 static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
81 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
219 	struct MPT3SAS_ADAPTER *ioc;  member
315 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_set_debug_level()  local
322 	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)  in _scsih_set_debug_level()
323 		ioc->logging_level = logging_level;  in _scsih_set_debug_level()
387 mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_port_by_id()  argument
397 	if (!ioc->multipath_on_hba)  in mpt3sas_get_port_by_id()
401 	    &ioc->port_table_list, list) {  in mpt3sas_get_port_by_id()
416 	if (!ioc->multipath_on_hba) {  in mpt3sas_get_port_by_id()
422 		ioc_info(ioc,  in mpt3sas_get_port_by_id()
426 		    &ioc->port_table_list);  in mpt3sas_get_port_by_id()
441 mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_vphy_by_phy()  argument
510 _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in _scsih_get_sas_address()  argument
519 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_get_sas_address()
521 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_sas_address()
531 		if ((handle <= ioc->sas_hba.num_phys) &&  in _scsih_get_sas_address()
534 			*sas_address = ioc->sas_hba.sas_address;  in _scsih_get_sas_address()
545 	ioc_err(ioc, "handle(0x%04x), ioc_status(0x%04x), failure at %s:%d/%s()!\n",  in _scsih_get_sas_address()
564 _scsih_determine_boot_device(struct MPT3SAS_ADAPTER *ioc, void *device,  in _scsih_determine_boot_device()  argument
576 	if (!ioc->is_driver_loading)  in _scsih_determine_boot_device()
580 	if (!ioc->bios_pg3.BiosVersion)  in _scsih_determine_boot_device()
603 	if (!ioc->req_boot_device.device) {  in _scsih_determine_boot_device()
606 		    (ioc->bios_pg2.ReqBootDeviceForm &  in _scsih_determine_boot_device()
608 		    &ioc->bios_pg2.RequestedBootDevice)) {  in _scsih_determine_boot_device()
609 			dinitprintk(ioc,  in _scsih_determine_boot_device()
610 				    ioc_info(ioc, "%s: req_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
612 			ioc->req_boot_device.device = device;  in _scsih_determine_boot_device()
613 			ioc->req_boot_device.channel = channel;  in _scsih_determine_boot_device()
617 	if (!ioc->req_alt_boot_device.device) {  in _scsih_determine_boot_device()
620 		    (ioc->bios_pg2.ReqAltBootDeviceForm &  in _scsih_determine_boot_device()
622 		    &ioc->bios_pg2.RequestedAltBootDevice)) {  in _scsih_determine_boot_device()
623 			dinitprintk(ioc,  in _scsih_determine_boot_device()
624 				    ioc_info(ioc, "%s: req_alt_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
626 			ioc->req_alt_boot_device.device = device;  in _scsih_determine_boot_device()
627 			ioc->req_alt_boot_device.channel = channel;  in _scsih_determine_boot_device()
631 	if (!ioc->current_boot_device.device) {  in _scsih_determine_boot_device()
634 		    (ioc->bios_pg2.CurrentBootDeviceForm &  in _scsih_determine_boot_device()
636 		    &ioc->bios_pg2.CurrentBootDevice)) {  in _scsih_determine_boot_device()
637 			dinitprintk(ioc,  in _scsih_determine_boot_device()
638 				    ioc_info(ioc, "%s: current_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
640 			ioc->current_boot_device.device = device;  in _scsih_determine_boot_device()
641 			ioc->current_boot_device.channel = channel;  in _scsih_determine_boot_device()
647 __mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_from_target()  argument
652 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_from_target()
662 mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_sdev_from_target()  argument
668 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_from_target()
669 	ret = __mpt3sas_get_sdev_from_target(ioc, tgt_priv);  in mpt3sas_get_sdev_from_target()
670 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_from_target()
676 __mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_pdev_from_target()  argument
681 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_from_target()
701 mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_pdev_from_target()  argument
707 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_from_target()
708 	ret = __mpt3sas_get_pdev_from_target(ioc, tgt_priv);  in mpt3sas_get_pdev_from_target()
709 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_from_target()
727 __mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_by_rphy()  argument
732 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_rphy()
734 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in __mpt3sas_get_sdev_by_rphy()
742 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {  in __mpt3sas_get_sdev_by_rphy()
763 __mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_by_addr()  argument
771 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_addr()
773 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in __mpt3sas_get_sdev_by_addr()
782 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {  in __mpt3sas_get_sdev_by_addr()
805 mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_sdev_by_addr()  argument
811 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_addr()
812 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in mpt3sas_get_sdev_by_addr()
814 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_addr()
820 __mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in __mpt3sas_get_sdev_by_handle()  argument
824 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_handle()
826 	list_for_each_entry(sas_device, &ioc->sas_device_list, list)  in __mpt3sas_get_sdev_by_handle()
830 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list)  in __mpt3sas_get_sdev_by_handle()
851 mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_get_sdev_by_handle()  argument
856 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_handle()
857 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in mpt3sas_get_sdev_by_handle()
858 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_handle()
871 _scsih_display_enclosure_chassis_info(struct MPT3SAS_ADAPTER *ioc,  in _scsih_display_enclosure_chassis_info()  argument
908 			ioc_info(ioc, "enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_display_enclosure_chassis_info()
912 			ioc_info(ioc, "enclosure level(0x%04x), connector name( %s)\n",  in _scsih_display_enclosure_chassis_info()
916 			ioc_info(ioc, "chassis slot(0x%04x)\n",  in _scsih_display_enclosure_chassis_info()
930 _scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_remove()  argument
937 	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_remove()
940 	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);  in _scsih_sas_device_remove()
946 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_remove()
951 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_remove()
960 _scsih_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_device_remove_by_handle()  argument
965 	if (ioc->shost_recovery)  in _scsih_device_remove_by_handle()
968 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_device_remove_by_handle()
969 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_device_remove_by_handle()
974 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_device_remove_by_handle()
976 		_scsih_remove_device(ioc, sas_device);  in _scsih_device_remove_by_handle()
991 mpt3sas_device_remove_by_sas_address(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_device_remove_by_sas_address()  argument
997 	if (ioc->shost_recovery)  in mpt3sas_device_remove_by_sas_address()
1000 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_device_remove_by_sas_address()
1001 	sas_device = __mpt3sas_get_sdev_by_addr(ioc, sas_address, port);  in mpt3sas_device_remove_by_sas_address()
1006 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_device_remove_by_sas_address()
1008 		_scsih_remove_device(ioc, sas_device);  in mpt3sas_device_remove_by_sas_address()
1022 _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_add()  argument
1027 	dewtprintk(ioc,  in _scsih_sas_device_add()
1028 		   ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_add()
1032 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_sas_device_add()
1035 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_add()
1037 	list_add_tail(&sas_device->list, &ioc->sas_device_list);  in _scsih_sas_device_add()
1038 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_add()
1040 	if (ioc->hide_drives) {  in _scsih_sas_device_add()
1041 		clear_bit(sas_device->handle, ioc->pend_os_device_add);  in _scsih_sas_device_add()
1045 	if (!mpt3sas_transport_port_add(ioc, sas_device->handle,  in _scsih_sas_device_add()
1047 		_scsih_sas_device_remove(ioc, sas_device);  in _scsih_sas_device_add()
1054 		if (!ioc->is_driver_loading) {  in _scsih_sas_device_add()
1055 			mpt3sas_transport_port_remove(ioc,  in _scsih_sas_device_add()
1059 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_sas_device_add()
1062 		clear_bit(sas_device->handle, ioc->pend_os_device_add);  in _scsih_sas_device_add()
1074 _scsih_sas_device_init_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_init_add()  argument
1079 	dewtprintk(ioc,  in _scsih_sas_device_init_add()
1080 		   ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_init_add()
1084 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_sas_device_init_add()
1087 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_init_add()
1089 	list_add_tail(&sas_device->list, &ioc->sas_device_init_list);  in _scsih_sas_device_init_add()
1090 	_scsih_determine_boot_device(ioc, sas_device, 0);  in _scsih_sas_device_init_add()
1091 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_init_add()
1096 __mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in __mpt3sas_get_pdev_by_wwid()  argument
1100 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_wwid()
1102 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_wwid()
1106 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_wwid()
1129 mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in mpt3sas_get_pdev_by_wwid()  argument
1134 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_wwid()
1135 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in mpt3sas_get_pdev_by_wwid()
1136 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_wwid()
1143 __mpt3sas_get_pdev_by_idchannel(struct MPT3SAS_ADAPTER *ioc, int id,  in __mpt3sas_get_pdev_by_idchannel()  argument
1148 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_idchannel()
1150 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_idchannel()
1154 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_idchannel()
1166 __mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in __mpt3sas_get_pdev_by_handle()  argument
1170 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_handle()
1172 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_handle()
1176 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_handle()
1200 mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_get_pdev_by_handle()  argument
1205 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_handle()
1206 	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in mpt3sas_get_pdev_by_handle()
1207 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_handle()
1222 _scsih_set_nvme_max_shutdown_latency(struct MPT3SAS_ADAPTER *ioc)  in _scsih_set_nvme_max_shutdown_latency()  argument
1228 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_set_nvme_max_shutdown_latency()
1229 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {  in _scsih_set_nvme_max_shutdown_latency()
1236 	ioc->max_shutdown_latency = shutdown_latency;  in _scsih_set_nvme_max_shutdown_latency()
1237 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_set_nvme_max_shutdown_latency()
1249 _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_remove()  argument
1258 	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove()
1261 		ioc_info(ioc, "removing enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove()
1265 		ioc_info(ioc, "removing enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove()
1269 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove()
1274 	if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)  in _scsih_pcie_device_remove()
1276 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove()
1288 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _scsih_pcie_device_remove()
1298 _scsih_pcie_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_device_remove_by_handle()  argument
1305 	if (ioc->shost_recovery)  in _scsih_pcie_device_remove_by_handle()
1308 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove_by_handle()
1309 	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_pcie_device_remove_by_handle()
1316 		if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)  in _scsih_pcie_device_remove_by_handle()
1319 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove_by_handle()
1321 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in _scsih_pcie_device_remove_by_handle()
1331 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _scsih_pcie_device_remove_by_handle()
1342 _scsih_pcie_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_add()  argument
1347 	dewtprintk(ioc,  in _scsih_pcie_device_add()
1348 		   ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_add()
1352 		dewtprintk(ioc,  in _scsih_pcie_device_add()
1353 			   ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",  in _scsih_pcie_device_add()
1358 		dewtprintk(ioc,  in _scsih_pcie_device_add()
1359 			   ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_add()
1363 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_add()
1365 	list_add_tail(&pcie_device->list, &ioc->pcie_device_list);  in _scsih_pcie_device_add()
1366 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_add()
1370 		clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1373 	if (scsi_add_device(ioc->shost, PCIE_CHANNEL, pcie_device->id, 0)) {  in _scsih_pcie_device_add()
1374 		_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_pcie_device_add()
1376 		if (!ioc->is_driver_loading) {  in _scsih_pcie_device_add()
1378 			clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1381 		clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1393 _scsih_pcie_device_init_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_init_add()  argument
1398 	dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1399 		   ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_init_add()
1403 		dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1404 			   ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",  in _scsih_pcie_device_init_add()
1409 		dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1410 			   ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_init_add()
1414 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_init_add()
1416 	list_add_tail(&pcie_device->list, &ioc->pcie_device_init_list);  in _scsih_pcie_device_init_add()
1419 		_scsih_determine_boot_device(ioc, pcie_device, PCIE_CHANNEL);  in _scsih_pcie_device_init_add()
1420 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_init_add()
1433 _scsih_raid_device_find_by_id(struct MPT3SAS_ADAPTER *ioc, int id, int channel)  in _scsih_raid_device_find_by_id()  argument
1438 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_raid_device_find_by_id()
1459 mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_raid_device_find_by_handle()  argument
1464 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in mpt3sas_raid_device_find_by_handle()
1485 _scsih_raid_device_find_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in _scsih_raid_device_find_by_wwid()  argument
1490 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_raid_device_find_by_wwid()
1509 _scsih_raid_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_raid_device_add()  argument
1514 	dewtprintk(ioc,  in _scsih_raid_device_add()
1515 		   ioc_info(ioc, "%s: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_raid_device_add()
1519 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_raid_device_add()
1520 	list_add_tail(&raid_device->list, &ioc->raid_device_list);  in _scsih_raid_device_add()
1521 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_raid_device_add()
1531 _scsih_raid_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_raid_device_remove()  argument
1536 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_raid_device_remove()
1539 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_raid_device_remove()
1552 mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_expander_find_by_handle()  argument
1557 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in mpt3sas_scsih_expander_find_by_handle()
1577 mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_enclosure_find_by_handle()  argument
1582 	list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {  in mpt3sas_scsih_enclosure_find_by_handle()
1602 mpt3sas_scsih_expander_find_by_sas_address(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_scsih_expander_find_by_sas_address()  argument
1610 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in mpt3sas_scsih_expander_find_by_sas_address()
1631 _scsih_expander_node_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_expander_node_add()  argument
1636 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_node_add()
1637 	list_add_tail(&sas_expander->list, &ioc->sas_expander_list);  in _scsih_expander_node_add()
1638 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_node_add()
1691 _scsih_scsi_lookup_find_by_target(struct MPT3SAS_ADAPTER *ioc, int id,  in _scsih_scsi_lookup_find_by_target()  argument
1698 	     smid <= ioc->shost->can_queue; smid++) {  in _scsih_scsi_lookup_find_by_target()
1699 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_scsi_lookup_find_by_target()
1721 _scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,  in _scsih_scsi_lookup_find_by_lun()  argument
1727 	for (smid = 1; smid <= ioc->shost->can_queue; smid++) {  in _scsih_scsi_lookup_find_by_lun()
1729 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_scsi_lookup_find_by_lun()
1749 mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_scsih_scsi_lookup_get()  argument
1757 	    smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) {  in mpt3sas_scsih_scsi_lookup_get()
1759 		    ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;  in mpt3sas_scsih_scsi_lookup_get()
1761 		mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_scsih_scsi_lookup_get()
1772 		scmd = scsi_host_find_tag(ioc->shost, unique_tag);  in mpt3sas_scsih_scsi_lookup_get()
1794 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_change_queue_depth()  local
1806 	if (ioc->enable_sdev_max_qd || ioc->is_gen35_ioc)  in scsih_change_queue_depth()
1818 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_change_queue_depth()
1819 	sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);  in scsih_change_queue_depth()
1826 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_change_queue_depth()
1853 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in mpt3sas_scsih_change_queue_depth()  local
1855 	if (ioc->enable_sdev_max_qd)  in mpt3sas_scsih_change_queue_depth()
1872 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_target_alloc()  local
1891 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_target_alloc()
1892 		raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,  in scsih_target_alloc()
1898 			if (ioc->is_warpdrive)  in scsih_target_alloc()
1902 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_target_alloc()
1908 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_target_alloc()
1909 		pcie_device = __mpt3sas_get_pdev_by_idchannel(ioc, starget->id,  in scsih_target_alloc()
1925 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_target_alloc()
1930 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_target_alloc()
1932 	sas_device = __mpt3sas_get_sdev_by_rphy(ioc, rphy);  in scsih_target_alloc()
1942 		if (test_bit(sas_device->handle, ioc->pd_handles))  in scsih_target_alloc()
1949 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_target_alloc()
1962 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_target_destroy()  local
1974 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_target_destroy()
1975 		raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,  in scsih_target_destroy()
1981 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_target_destroy()
1986 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_target_destroy()
1987 		pcie_device = __mpt3sas_get_pdev_from_target(ioc,  in scsih_target_destroy()
2002 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_target_destroy()
2006 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_target_destroy()
2007 	sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);  in scsih_target_destroy()
2022 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_target_destroy()
2040 	struct MPT3SAS_ADAPTER *ioc;  in scsih_slave_alloc()  local
2066 	ioc = shost_priv(shost);  in scsih_slave_alloc()
2068 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_slave_alloc()
2069 		raid_device = _scsih_raid_device_find_by_id(ioc,  in scsih_slave_alloc()
2073 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_slave_alloc()
2076 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_alloc()
2077 		pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,  in scsih_slave_alloc()
2088 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_alloc()
2091 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_alloc()
2092 		sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in scsih_slave_alloc()
2105 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_alloc()
2121 	struct MPT3SAS_ADAPTER *ioc;  in scsih_slave_destroy()  local
2134 	ioc = shost_priv(shost);  in scsih_slave_destroy()
2137 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_destroy()
2138 		pcie_device = __mpt3sas_get_pdev_from_target(ioc,  in scsih_slave_destroy()
2146 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_destroy()
2149 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_destroy()
2150 		sas_device = __mpt3sas_get_sdev_from_target(ioc,  in scsih_slave_destroy()
2157 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_destroy()
2171 _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,  in _scsih_display_sata_capabilities()  argument
2180 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_display_sata_capabilities()
2182 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_display_sata_capabilities()
2190 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_display_sata_capabilities()
2225 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_is_raid()  local
2227 	if (ioc->is_warpdrive)  in scsih_is_raid()
2248 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_get_resync()  local
2259 	if (ioc->is_warpdrive)  in scsih_get_resync()
2262 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_get_resync()
2263 	raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,  in scsih_get_resync()
2269 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_get_resync()
2274 	if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,  in scsih_get_resync()
2277 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in scsih_get_resync()
2290 	switch (ioc->hba_mpi_version_belonged) {  in scsih_get_resync()
2309 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_get_state()  local
2318 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_get_state()
2319 	raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,  in scsih_get_state()
2323 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_get_state()
2328 	if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,  in scsih_get_state()
2331 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in scsih_get_state()
2356 	switch (ioc->hba_mpi_version_belonged) {  in scsih_get_state()
2374 _scsih_set_level(struct MPT3SAS_ADAPTER *ioc,  in _scsih_set_level()  argument
2394 	switch (ioc->hba_mpi_version_belonged) {  in _scsih_set_level()
2416 _scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,  in _scsih_get_volume_capabilities()  argument
2426 	if ((mpt3sas_config_get_number_pds(ioc, raid_device->handle,  in _scsih_get_volume_capabilities()
2428 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2429 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2439 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2440 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2445 	if ((mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, vol_pg0,  in _scsih_get_volume_capabilities()
2447 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2448 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2459 	if (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_get_volume_capabilities()
2462 		if (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_get_volume_capabilities()
2484 _scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)  in _scsih_enable_tlr()  argument
2491 	if (!(ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR))  in _scsih_enable_tlr()
2512 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_slave_configure()  local
2536 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_slave_configure()
2537 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in scsih_slave_configure()
2538 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_slave_configure()
2540 			dfailprintk(ioc,  in scsih_slave_configure()
2541 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2546 		if (_scsih_get_volume_capabilities(ioc, raid_device)) {  in scsih_slave_configure()
2547 			dfailprintk(ioc,  in scsih_slave_configure()
2548 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2556 		mpt3sas_init_warpdrive_properties(ioc, raid_device);  in scsih_slave_configure()
2582 			if (ioc->manu_pg10.OEMIdentifier &&  in scsih_slave_configure()
2583 			    (le32_to_cpu(ioc->manu_pg10.GenericFlags0) &  in scsih_slave_configure()
2605 		if (!ioc->hide_ir_msg)  in scsih_slave_configure()
2624 		if (!ioc->is_warpdrive)  in scsih_slave_configure()
2625 			_scsih_set_level(ioc, sdev, raid_device->volume_type);  in scsih_slave_configure()
2631 		if (mpt3sas_config_get_volume_handle(ioc, handle,  in scsih_slave_configure()
2633 			dfailprintk(ioc,  in scsih_slave_configure()
2634 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2638 		if (volume_handle && mpt3sas_config_get_volume_wwid(ioc,  in scsih_slave_configure()
2640 			dfailprintk(ioc,  in scsih_slave_configure()
2641 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2649 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2650 		pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,  in scsih_slave_configure()
2653 			spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2654 			dfailprintk(ioc,  in scsih_slave_configure()
2655 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2660 		qdepth = ioc->max_nvme_qd;  in scsih_slave_configure()
2684 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2693 				ioc->page_size - 1);  in scsih_slave_configure()
2697 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2698 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in scsih_slave_configure()
2702 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2703 		dfailprintk(ioc,  in scsih_slave_configure()
2704 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2713 			ioc->max_wideport_qd : ioc->max_narrowport_qd;  in scsih_slave_configure()
2725 		qdepth = ioc->max_sata_qd;  in scsih_slave_configure()
2741 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2744 		_scsih_display_sata_capabilities(ioc, handle, sdev);  in scsih_slave_configure()
2751 		_scsih_enable_tlr(ioc, sdev);  in scsih_slave_configure()
2809 _scsih_response_code(struct MPT3SAS_ADAPTER *ioc, u8 response_code)  in _scsih_response_code()  argument
2842 	ioc_warn(ioc, "response_code(0x%01x): %s\n", response_code, desc);  in _scsih_response_code()
2859 _scsih_tm_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_tm_done()  argument
2863 	if (ioc->tm_cmds.status == MPT3_CMD_NOT_USED)  in _scsih_tm_done()
2865 	if (ioc->tm_cmds.smid != smid)  in _scsih_tm_done()
2867 	ioc->tm_cmds.status |= MPT3_CMD_COMPLETE;  in _scsih_tm_done()
2868 	mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_done()
2870 		memcpy(ioc->tm_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in _scsih_tm_done()
2871 		ioc->tm_cmds.status |= MPT3_CMD_REPLY_VALID;  in _scsih_tm_done()
2873 	ioc->tm_cmds.status &= ~MPT3_CMD_PENDING;  in _scsih_tm_done()
2874 	complete(&ioc->tm_cmds.done);  in _scsih_tm_done()
2886 mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_set_tm_flag()  argument
2892 	shost_for_each_device(sdev, ioc->shost) {  in mpt3sas_scsih_set_tm_flag()
2901 			ioc->ignore_loginfos = 1;  in mpt3sas_scsih_set_tm_flag()
2914 mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_clear_tm_flag()  argument
2920 	shost_for_each_device(sdev, ioc->shost) {  in mpt3sas_scsih_clear_tm_flag()
2929 			ioc->ignore_loginfos = 0;  in mpt3sas_scsih_clear_tm_flag()
2947 scsih_tm_cmd_map_status(struct MPT3SAS_ADAPTER *ioc, uint channel,  in scsih_tm_cmd_map_status()  argument
2951 	if (smid_task <= ioc->shost->can_queue) {  in scsih_tm_cmd_map_status()
2954 			if (!(_scsih_scsi_lookup_find_by_target(ioc,  in scsih_tm_cmd_map_status()
2960 			if (!(_scsih_scsi_lookup_find_by_lun(ioc, id,  in scsih_tm_cmd_map_status()
2967 	} else if (smid_task == ioc->scsih_cmds.smid) {  in scsih_tm_cmd_map_status()
2968 		if ((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) ||  in scsih_tm_cmd_map_status()
2969 		    (ioc->scsih_cmds.status & MPT3_CMD_NOT_USED))  in scsih_tm_cmd_map_status()
2971 	} else if (smid_task == ioc->ctl_cmds.smid) {  in scsih_tm_cmd_map_status()
2972 		if ((ioc->ctl_cmds.status & MPT3_CMD_COMPLETE) ||  in scsih_tm_cmd_map_status()
2973 		    (ioc->ctl_cmds.status & MPT3_CMD_NOT_USED))  in scsih_tm_cmd_map_status()
2998 scsih_tm_post_processing(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in scsih_tm_post_processing()  argument
3003 	rc = scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);  in scsih_tm_post_processing()
3007 	ioc_info(ioc,  in scsih_tm_post_processing()
3017 	mpt3sas_base_mask_interrupts(ioc);  in scsih_tm_post_processing()
3018 	mpt3sas_base_sync_reply_irqs(ioc, 1);  in scsih_tm_post_processing()
3019 	mpt3sas_base_unmask_interrupts(ioc);  in scsih_tm_post_processing()
3021 	return scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);  in scsih_tm_post_processing()
3046 mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,  in mpt3sas_scsih_issue_tm()  argument
3058 	lockdep_assert_held(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_tm()
3060 	if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_scsih_issue_tm()
3061 		ioc_info(ioc, "%s: tm_cmd busy!!!\n", __func__);  in mpt3sas_scsih_issue_tm()
3065 	if (ioc->shost_recovery || ioc->remove_host ||  in mpt3sas_scsih_issue_tm()
3066 	    ioc->pci_error_recovery) {  in mpt3sas_scsih_issue_tm()
3067 		ioc_info(ioc, "%s: host reset in progress!\n", __func__);  in mpt3sas_scsih_issue_tm()
3071 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_scsih_issue_tm()
3073 		dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));  in mpt3sas_scsih_issue_tm()
3074 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3079 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_scsih_issue_tm()
3081 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3085 		mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_scsih_issue_tm()
3087 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3091 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_cb_idx);  in mpt3sas_scsih_issue_tm()
3093 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_scsih_issue_tm()
3097 	dtmprintk(ioc,  in mpt3sas_scsih_issue_tm()
3098 …ioc_info(ioc, "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%…  in mpt3sas_scsih_issue_tm()
3100 	ioc->tm_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_scsih_issue_tm()
3101 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_scsih_issue_tm()
3102 	ioc->tm_cmds.smid = smid;  in mpt3sas_scsih_issue_tm()
3104 	memset(ioc->tm_cmds.reply, 0, sizeof(Mpi2SCSITaskManagementReply_t));  in mpt3sas_scsih_issue_tm()
3113 	mpt3sas_scsih_set_tm_flag(ioc, handle);  in mpt3sas_scsih_issue_tm()
3114 	init_completion(&ioc->tm_cmds.done);  in mpt3sas_scsih_issue_tm()
3115 	ioc->put_smid_hi_priority(ioc, smid, msix_task);  in mpt3sas_scsih_issue_tm()
3116 	wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);  in mpt3sas_scsih_issue_tm()
3117 	if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_scsih_issue_tm()
3118 		mpt3sas_check_cmd_timeout(ioc,  in mpt3sas_scsih_issue_tm()
3119 		    ioc->tm_cmds.status, mpi_request,  in mpt3sas_scsih_issue_tm()
3122 			rc = mpt3sas_base_hard_reset_handler(ioc,  in mpt3sas_scsih_issue_tm()
3130 	mpt3sas_base_sync_reply_irqs(ioc, 0);  in mpt3sas_scsih_issue_tm()
3132 	if (ioc->tm_cmds.status & MPT3_CMD_REPLY_VALID) {  in mpt3sas_scsih_issue_tm()
3133 		mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);  in mpt3sas_scsih_issue_tm()
3134 		mpi_reply = ioc->tm_cmds.reply;  in mpt3sas_scsih_issue_tm()
3135 		dtmprintk(ioc,  in mpt3sas_scsih_issue_tm()
3136 			  ioc_info(ioc, "complete tm: ioc_status(0x%04x), loginfo(0x%08x), term_count(0x%08x)\n",  in mpt3sas_scsih_issue_tm()
3140 		if (ioc->logging_level & MPT_DEBUG_TM) {  in mpt3sas_scsih_issue_tm()
3141 			_scsih_response_code(ioc, mpi_reply->ResponseCode);  in mpt3sas_scsih_issue_tm()
3162 		request = mpt3sas_base_get_msg_frame(ioc, smid_task);  in mpt3sas_scsih_issue_tm()
3166 		ioc_info(ioc, "Task abort tm failed: handle(0x%04x),"  in mpt3sas_scsih_issue_tm()
3175 		rc = scsih_tm_post_processing(ioc, handle, channel, id, lun,  in mpt3sas_scsih_issue_tm()
3187 	mpt3sas_scsih_clear_tm_flag(ioc, handle);  in mpt3sas_scsih_issue_tm()
3188 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_scsih_issue_tm()
3192 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in mpt3sas_scsih_issue_locked_tm()  argument
3198 	mutex_lock(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_locked_tm()
3199 	ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,  in mpt3sas_scsih_issue_locked_tm()
3201 	mutex_unlock(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_locked_tm()
3214 _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)  in _scsih_tm_display_info()  argument
3225 	if (ioc->hide_ir_msg)  in _scsih_tm_display_info()
3238 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_tm_display_info()
3239 		pcie_device = __mpt3sas_get_pdev_from_target(ioc, priv_target);  in _scsih_tm_display_info()
3259 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_tm_display_info()
3262 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_tm_display_info()
3263 		sas_device = __mpt3sas_get_sdev_from_target(ioc, priv_target);  in _scsih_tm_display_info()
3284 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_tm_display_info()
3297 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_abort()  local
3309 	_scsih_tm_display_info(ioc, scmd);  in scsih_abort()
3313 	    ioc->remove_host) {  in scsih_abort()
3340 	mpt3sas_halt_firmware(ioc);  in scsih_abort()
3343 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_abort()
3344 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_abort()
3346 		timeout = ioc->nvme_abort_timeout;  in scsih_abort()
3347 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_abort()
3371 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_dev_reset()  local
3385 	_scsih_tm_display_info(ioc, scmd);  in scsih_dev_reset()
3389 	    ioc->remove_host) {  in scsih_dev_reset()
3402 		sas_device = mpt3sas_get_sdev_from_target(ioc,  in scsih_dev_reset()
3415 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_dev_reset()
3417 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_dev_reset()
3424 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_dev_reset()
3452 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_target_reset()  local
3465 	_scsih_tm_display_info(ioc, scmd);  in scsih_target_reset()
3469 	    ioc->remove_host) {  in scsih_target_reset()
3482 		sas_device = mpt3sas_get_sdev_from_target(ioc,  in scsih_target_reset()
3495 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_target_reset()
3497 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_target_reset()
3503 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_target_reset()
3531 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_host_reset()  local
3534 	ioc_info(ioc, "attempting host reset! scmd(0x%p)\n", scmd);  in scsih_host_reset()
3537 	if (ioc->is_driver_loading || ioc->remove_host) {  in scsih_host_reset()
3538 		ioc_info(ioc, "Blocking the host reset\n");  in scsih_host_reset()
3543 	retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in scsih_host_reset()
3546 	ioc_info(ioc, "host reset: %s scmd(0x%p)\n",  in scsih_host_reset()
3562 _scsih_fw_event_add(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)  in _scsih_fw_event_add()  argument
3566 	if (ioc->firmware_event_thread == NULL)  in _scsih_fw_event_add()
3569 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_fw_event_add()
3572 	list_add_tail(&fw_event->list, &ioc->fw_event_list);  in _scsih_fw_event_add()
3575 	queue_work(ioc->firmware_event_thread, &fw_event->work);  in _scsih_fw_event_add()
3576 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_fw_event_add()
3588 _scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work  in _scsih_fw_event_del_from_list()  argument
3593 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_fw_event_del_from_list()
3598 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_fw_event_del_from_list()
3608 mpt3sas_send_trigger_data_event(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_send_trigger_data_event()  argument
3614 	if (ioc->is_driver_loading)  in mpt3sas_send_trigger_data_event()
3621 	fw_event->ioc = ioc;  in mpt3sas_send_trigger_data_event()
3623 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_send_trigger_data_event()
3632 _scsih_error_recovery_delete_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_error_recovery_delete_devices()  argument
3640 	fw_event->ioc = ioc;  in _scsih_error_recovery_delete_devices()
3641 	_scsih_fw_event_add(ioc, fw_event);  in _scsih_error_recovery_delete_devices()
3650 mpt3sas_port_enable_complete(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_port_enable_complete()  argument
3658 	fw_event->ioc = ioc;  in mpt3sas_port_enable_complete()
3659 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_port_enable_complete()
3663 static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc)  in dequeue_next_fw_event()  argument
3668 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in dequeue_next_fw_event()
3669 	if (!list_empty(&ioc->fw_event_list)) {  in dequeue_next_fw_event()
3670 		fw_event = list_first_entry(&ioc->fw_event_list,  in dequeue_next_fw_event()
3675 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in dequeue_next_fw_event()
3690 _scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc)  in _scsih_fw_event_cleanup_queue()  argument
3694 	if ((list_empty(&ioc->fw_event_list) && !ioc->current_event) ||  in _scsih_fw_event_cleanup_queue()
3695 	    !ioc->firmware_event_thread)  in _scsih_fw_event_cleanup_queue()
3703 	if (ioc->shost_recovery && ioc->current_event)  in _scsih_fw_event_cleanup_queue()
3704 		ioc->current_event->ignore = 1;  in _scsih_fw_event_cleanup_queue()
3706 	ioc->fw_events_cleanup = 1;  in _scsih_fw_event_cleanup_queue()
3707 	while ((fw_event = dequeue_next_fw_event(ioc)) ||  in _scsih_fw_event_cleanup_queue()
3708 	     (fw_event = ioc->current_event)) {  in _scsih_fw_event_cleanup_queue()
3724 		if (fw_event == ioc->current_event &&  in _scsih_fw_event_cleanup_queue()
3725 		    ioc->current_event->event !=  in _scsih_fw_event_cleanup_queue()
3727 			ioc->current_event = NULL;  in _scsih_fw_event_cleanup_queue()
3740 			ioc->port_enable_cmds.status |= MPT3_CMD_RESET;  in _scsih_fw_event_cleanup_queue()
3741 			ioc->start_scan = 0;  in _scsih_fw_event_cleanup_queue()
3756 	ioc->fw_events_cleanup = 0;  in _scsih_fw_event_cleanup_queue()
3835 _scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)  in _scsih_ublock_io_all_device()  argument
3840 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_ublock_io_all_device()
3847 		dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,  in _scsih_ublock_io_all_device()
3864 _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_ublock_io_device()  argument
3870 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_ublock_io_device()
3892 _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)  in _scsih_block_io_all_device()  argument
3897 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_block_io_all_device()
3921 _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_block_io_device()  argument
3927 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_block_io_device()
3929 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_block_io_device()
3962 _scsih_block_io_to_children_attached_to_ex(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_children_attached_to_ex()  argument
3977 			spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_block_io_to_children_attached_to_ex()
3978 			sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_block_io_to_children_attached_to_ex()
3983 						ioc->blocking_handles);  in _scsih_block_io_to_children_attached_to_ex()
3986 			spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_block_io_to_children_attached_to_ex()
3999 			    ioc, mpt3sas_port->remote_identify.sas_address,  in _scsih_block_io_to_children_attached_to_ex()
4001 			_scsih_block_io_to_children_attached_to_ex(ioc,  in _scsih_block_io_to_children_attached_to_ex()
4016 _scsih_block_io_to_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_children_attached_directly()  argument
4030 			_scsih_block_io_device(ioc, handle);  in _scsih_block_io_to_children_attached_directly()
4043 _scsih_block_io_to_pcie_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_pcie_children_attached_directly()  argument
4058 			_scsih_block_io_device(ioc, handle);  in _scsih_block_io_to_pcie_children_attached_directly()
4077 _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_tm_tr_send()  argument
4091 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_send()
4092 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4093 			   ioc_info(ioc, "%s: host in pci error recovery: handle(0x%04x)\n",  in _scsih_tm_tr_send()
4097 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_tm_tr_send()
4099 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4100 			   ioc_info(ioc, "%s: host is not operational: handle(0x%04x)\n",  in _scsih_tm_tr_send()
4106 	if (test_bit(handle, ioc->pd_handles))  in _scsih_tm_tr_send()
4109 	clear_bit(handle, ioc->pend_os_device_add);  in _scsih_tm_tr_send()
4111 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_tm_tr_send()
4112 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_tm_tr_send()
4120 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_tm_tr_send()
4122 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_tm_tr_send()
4123 		pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_tm_tr_send()
4130 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_tm_tr_send()
4131 		if (pcie_device && (!ioc->tm_custom_handling) &&  in _scsih_tm_tr_send()
4140 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4141 			   ioc_info(ioc, "setting delete flag: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_tm_tr_send()
4145 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4146 					   ioc_info(ioc, "setting delete flag:enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_tm_tr_send()
4150 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4151 					   ioc_info(ioc, "setting delete flag: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_tm_tr_send()
4156 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4157 					   ioc_info(ioc, "setting delete flag: logical id(0x%016llx), slot(%d)\n",  in _scsih_tm_tr_send()
4161 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4162 					   ioc_info(ioc, "setting delete flag:, enclosure level(0x%04x), connector name( %s)\n",  in _scsih_tm_tr_send()
4166 		_scsih_ublock_io_device(ioc, sas_address, port);  in _scsih_tm_tr_send()
4170 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_cb_idx);  in _scsih_tm_tr_send()
4177 		list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);  in _scsih_tm_tr_send()
4178 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4179 			   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_tm_tr_send()
4184 	dewtprintk(ioc,  in _scsih_tm_tr_send()
4185 		   ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_send()
4186 			    handle, smid, ioc->tm_tr_cb_idx));  in _scsih_tm_tr_send()
4187 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_send()
4193 	set_bit(handle, ioc->device_remove_in_progress);  in _scsih_tm_tr_send()
4194 	ioc->put_smid_hi_priority(ioc, smid, 0);  in _scsih_tm_tr_send()
4195 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);  in _scsih_tm_tr_send()
4221 _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in _scsih_tm_tr_complete()  argument
4227 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_tr_complete()
4233 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_complete()
4234 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4235 			   ioc_info(ioc, "%s: host in pci error recovery\n",  in _scsih_tm_tr_complete()
4239 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_tm_tr_complete()
4241 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4242 			   ioc_info(ioc, "%s: host is not operational\n",  in _scsih_tm_tr_complete()
4247 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_tm_tr_complete()
4251 	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_complete()
4254 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4255 			   ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",  in _scsih_tm_tr_complete()
4261 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);  in _scsih_tm_tr_complete()
4262 	dewtprintk(ioc,  in _scsih_tm_tr_complete()
4263 …ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), c…  in _scsih_tm_tr_complete()
4268 	smid_sas_ctrl = mpt3sas_base_get_smid(ioc, ioc->tm_sas_control_cb_idx);  in _scsih_tm_tr_complete()
4272 			return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4275 		list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);  in _scsih_tm_tr_complete()
4276 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4277 			   ioc_info(ioc, "DELAYED:sc:handle(0x%04x), (open)\n",  in _scsih_tm_tr_complete()
4279 		return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4282 	dewtprintk(ioc,  in _scsih_tm_tr_complete()
4283 		   ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_complete()
4284 			    handle, smid_sas_ctrl, ioc->tm_sas_control_cb_idx));  in _scsih_tm_tr_complete()
4285 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid_sas_ctrl);  in _scsih_tm_tr_complete()
4290 	ioc->put_smid_default(ioc, smid_sas_ctrl);  in _scsih_tm_tr_complete()
4292 	return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4302 inline bool _scsih_allow_scmd_to_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_allow_scmd_to_device()  argument
4306 	if (ioc->pci_error_recovery)  in _scsih_allow_scmd_to_device()
4309 	if (ioc->hba_mpi_version_belonged == MPI2_VERSION) {  in _scsih_allow_scmd_to_device()
4310 		if (ioc->remove_host)  in _scsih_allow_scmd_to_device()
4316 	if (ioc->remove_host) {  in _scsih_allow_scmd_to_device()
4346 _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _scsih_sas_control_complete()  argument
4350 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_sas_control_complete()
4353 		dewtprintk(ioc,  in _scsih_sas_control_complete()
4354 …ioc_info(ioc, "sc_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_sas_control_complete()
4361 			    ioc->device_remove_in_progress);  in _scsih_sas_control_complete()
4364 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_sas_control_complete()
4367 	return mpt3sas_check_for_pending_internal_cmds(ioc, smid);  in _scsih_sas_control_complete()
4381 _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_tm_tr_volume_send()  argument
4387 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_volume_send()
4388 		dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4389 			   ioc_info(ioc, "%s: host reset in progress!\n",  in _scsih_tm_tr_volume_send()
4394 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_volume_cb_idx);  in _scsih_tm_tr_volume_send()
4401 		list_add_tail(&delayed_tr->list, &ioc->delayed_tr_volume_list);  in _scsih_tm_tr_volume_send()
4402 		dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4403 			   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_tm_tr_volume_send()
4408 	dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4409 		   ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_volume_send()
4410 			    handle, smid, ioc->tm_tr_volume_cb_idx));  in _scsih_tm_tr_volume_send()
4411 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_volume_send()
4416 	ioc->put_smid_hi_priority(ioc, smid, 0);  in _scsih_tm_tr_volume_send()
4431 _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _scsih_tm_volume_tr_complete()  argument
4437 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_volume_tr_complete()
4439 	if (ioc->shost_recovery || ioc->pci_error_recovery) {  in _scsih_tm_volume_tr_complete()
4440 		dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4441 			   ioc_info(ioc, "%s: host reset in progress!\n",  in _scsih_tm_volume_tr_complete()
4446 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_tm_volume_tr_complete()
4451 	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_volume_tr_complete()
4454 		dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4455 			   ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",  in _scsih_tm_volume_tr_complete()
4461 	dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4462 …ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), c…  in _scsih_tm_volume_tr_complete()
4467 	return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_volume_tr_complete()
4480 _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,  in _scsih_issue_delayed_event_ack()  argument
4484 	int i = smid - ioc->internal_smid;  in _scsih_issue_delayed_event_ack()
4491 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_event_ack()
4492 	ioc->internal_lookup[i].cb_idx = ioc->base_cb_idx;  in _scsih_issue_delayed_event_ack()
4493 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_event_ack()
4495 	dewtprintk(ioc,  in _scsih_issue_delayed_event_ack()
4496 		   ioc_info(ioc, "EVENT ACK: event(0x%04x), smid(%d), cb(%d)\n",  in _scsih_issue_delayed_event_ack()
4497 			    le16_to_cpu(event), smid, ioc->base_cb_idx));  in _scsih_issue_delayed_event_ack()
4498 	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_issue_delayed_event_ack()
4505 	ioc->put_smid_default(ioc, smid);  in _scsih_issue_delayed_event_ack()
4518 _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()  argument
4523 	int i = smid - ioc->internal_smid;  in _scsih_issue_delayed_sas_io_unit_ctrl()
4526 	if (ioc->remove_host) {  in _scsih_issue_delayed_sas_io_unit_ctrl()
4527 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4528 			   ioc_info(ioc, "%s: host has been removed\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4531 	} else if (ioc->pci_error_recovery) {  in _scsih_issue_delayed_sas_io_unit_ctrl()
4532 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4533 			   ioc_info(ioc, "%s: host in pci error recovery\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4537 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4539 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4540 			   ioc_info(ioc, "%s: host is not operational\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4549 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4550 	ioc->internal_lookup[i].cb_idx = ioc->tm_sas_control_cb_idx;  in _scsih_issue_delayed_sas_io_unit_ctrl()
4551 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4553 	dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4554 		   ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4555 			    handle, smid, ioc->tm_sas_control_cb_idx));  in _scsih_issue_delayed_sas_io_unit_ctrl()
4556 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4561 	ioc->put_smid_default(ioc, smid);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4578 mpt3sas_check_for_pending_internal_cmds(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_check_for_pending_internal_cmds()  argument
4583 	if (!list_empty(&ioc->delayed_event_ack_list)) {  in mpt3sas_check_for_pending_internal_cmds()
4584 		delayed_event_ack = list_entry(ioc->delayed_event_ack_list.next,  in mpt3sas_check_for_pending_internal_cmds()
4586 		_scsih_issue_delayed_event_ack(ioc, smid,  in mpt3sas_check_for_pending_internal_cmds()
4593 	if (!list_empty(&ioc->delayed_sc_list)) {  in mpt3sas_check_for_pending_internal_cmds()
4594 		delayed_sc = list_entry(ioc->delayed_sc_list.next,  in mpt3sas_check_for_pending_internal_cmds()
4596 		_scsih_issue_delayed_sas_io_unit_ctrl(ioc, smid,  in mpt3sas_check_for_pending_internal_cmds()
4617 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _scsih_check_for_pending_tm()  argument
4621 	if (!list_empty(&ioc->delayed_tr_volume_list)) {  in _scsih_check_for_pending_tm()
4622 		delayed_tr = list_entry(ioc->delayed_tr_volume_list.next,  in _scsih_check_for_pending_tm()
4624 		mpt3sas_base_free_smid(ioc, smid);  in _scsih_check_for_pending_tm()
4625 		_scsih_tm_tr_volume_send(ioc, delayed_tr->handle);  in _scsih_check_for_pending_tm()
4631 	if (!list_empty(&ioc->delayed_tr_list)) {  in _scsih_check_for_pending_tm()
4632 		delayed_tr = list_entry(ioc->delayed_tr_list.next,  in _scsih_check_for_pending_tm()
4634 		mpt3sas_base_free_smid(ioc, smid);  in _scsih_check_for_pending_tm()
4635 		_scsih_tm_tr_send(ioc, delayed_tr->handle);  in _scsih_check_for_pending_tm()
4656 _scsih_check_topo_delete_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_topo_delete_events()  argument
4674 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_topo_delete_events()
4678 	if (expander_handle < ioc->sas_hba.num_phys) {  in _scsih_check_topo_delete_events()
4679 		_scsih_block_io_to_children_attached_directly(ioc, event_data);  in _scsih_check_topo_delete_events()
4685 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_check_topo_delete_events()
4686 		sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,  in _scsih_check_topo_delete_events()
4688 		_scsih_block_io_to_children_attached_to_ex(ioc, sas_expander);  in _scsih_check_topo_delete_events()
4689 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_check_topo_delete_events()
4691 			handle = find_first_bit(ioc->blocking_handles,  in _scsih_check_topo_delete_events()
4692 			    ioc->facts.MaxDevHandle);  in _scsih_check_topo_delete_events()
4693 			if (handle < ioc->facts.MaxDevHandle)  in _scsih_check_topo_delete_events()
4694 				_scsih_block_io_device(ioc, handle);  in _scsih_check_topo_delete_events()
4695 		} while (test_and_clear_bit(handle, ioc->blocking_handles));  in _scsih_check_topo_delete_events()
4697 		_scsih_block_io_to_children_attached_directly(ioc, event_data);  in _scsih_check_topo_delete_events()
4703 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_check_topo_delete_events()
4704 	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {  in _scsih_check_topo_delete_events()
4716 				dewtprintk(ioc,  in _scsih_check_topo_delete_events()
4717 					   ioc_info(ioc, "setting ignoring flag\n"));  in _scsih_check_topo_delete_events()
4722 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_check_topo_delete_events()
4737 _scsih_check_pcie_topo_remove_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_pcie_topo_remove_events()  argument
4753 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_pcie_topo_remove_events()
4759 							ioc, event_data);  in _scsih_check_pcie_topo_remove_events()
4768 							ioc, event_data);  in _scsih_check_pcie_topo_remove_events()
4774 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_check_pcie_topo_remove_events()
4775 	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {  in _scsih_check_pcie_topo_remove_events()
4788 				dewtprintk(ioc,  in _scsih_check_pcie_topo_remove_events()
4789 					   ioc_info(ioc, "setting ignoring flag for switch event\n"));  in _scsih_check_pcie_topo_remove_events()
4794 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_check_pcie_topo_remove_events()
4805 _scsih_set_volume_delete_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_set_volume_delete_flag()  argument
4811 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_set_volume_delete_flag()
4812 	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_set_volume_delete_flag()
4818 		dewtprintk(ioc,  in _scsih_set_volume_delete_flag()
4819 			   ioc_info(ioc, "setting delete flag: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_set_volume_delete_flag()
4822 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_set_volume_delete_flag()
4859 _scsih_check_ir_config_unhide_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_ir_config_unhide_events()  argument
4870 	if (ioc->is_warpdrive)  in _scsih_check_ir_config_unhide_events()
4884 			_scsih_set_volume_delete_flag(ioc, volume_handle);  in _scsih_check_ir_config_unhide_events()
4902 		_scsih_tm_tr_volume_send(ioc, a);  in _scsih_check_ir_config_unhide_events()
4904 		_scsih_tm_tr_volume_send(ioc, b);  in _scsih_check_ir_config_unhide_events()
4913 		clear_bit(handle, ioc->pd_handles);  in _scsih_check_ir_config_unhide_events()
4915 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_ir_config_unhide_events()
4921 			list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);  in _scsih_check_ir_config_unhide_events()
4922 			dewtprintk(ioc,  in _scsih_check_ir_config_unhide_events()
4923 				   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_check_ir_config_unhide_events()
4926 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_ir_config_unhide_events()
4942 _scsih_check_volume_delete_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_volume_delete_events()  argument
4952 		_scsih_set_volume_delete_flag(ioc,  in _scsih_check_volume_delete_events()
4963 _scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_temp_threshold_events()  argument
4967 	if (ioc->temp_sensors_count >= event_data->SensorNum) {  in _scsih_temp_threshold_events()
4968 		ioc_err(ioc, "Temperature Threshold flags %s%s%s%s exceeded for Sensor: %d !!!\n",  in _scsih_temp_threshold_events()
4974 		ioc_err(ioc, "Current Temp In Celsius: %d\n",  in _scsih_temp_threshold_events()
4976 		if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _scsih_temp_threshold_events()
4977 			doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _scsih_temp_threshold_events()
4980 				mpt3sas_print_fault_code(ioc,  in _scsih_temp_threshold_events()
4984 				mpt3sas_print_coredump_info(ioc,  in _scsih_temp_threshold_events()
5013 _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)  in _scsih_flush_running_cmds()  argument
5020 	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {  in _scsih_flush_running_cmds()
5021 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_flush_running_cmds()
5027 		mpt3sas_base_clear_st(ioc, st);  in _scsih_flush_running_cmds()
5029 		if (ioc->pci_error_recovery || ioc->remove_host)  in _scsih_flush_running_cmds()
5035 	dtmprintk(ioc, ioc_info(ioc, "completing %d cmds\n", count));  in _scsih_flush_running_cmds()
5047 _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,  in _scsih_setup_eedp()  argument
5080 	if (ioc->is_gen35_ioc)  in _scsih_setup_eedp()
5127 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_qcmd()  local
5139 	if (ioc->logging_level & MPT_DEBUG_SCSI)  in scsih_qcmd()
5149 	if (!(_scsih_allow_scmd_to_device(ioc, scmd))) {  in scsih_qcmd()
5179 	if (ioc->shost_recovery || ioc->ioc_link_reset_in_progress) {  in scsih_qcmd()
5221 	if (((!ioc->is_warpdrive && !scsih_is_raid(&scmd->device->sdev_gendev))  in scsih_qcmd()
5226 	smid = mpt3sas_base_get_smid_scsiio(ioc, ioc->scsi_io_cb_idx, scmd);  in scsih_qcmd()
5228 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in scsih_qcmd()
5232 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in scsih_qcmd()
5233 	memset(mpi_request, 0, ioc->request_sz);  in scsih_qcmd()
5234 	_scsih_setup_eedp(ioc, scmd, mpi_request);  in scsih_qcmd()
5251 	    mpt3sas_base_get_sense_buffer_dma(ioc, smid);  in scsih_qcmd()
5259 		if (ioc->build_sg_scmd(ioc, scmd, smid, pcie_device)) {  in scsih_qcmd()
5260 			mpt3sas_base_free_smid(ioc, smid);  in scsih_qcmd()
5265 		ioc->build_zero_len_sge(ioc, &mpi_request->SGL);  in scsih_qcmd()
5269 		mpt3sas_setup_direct_io(ioc, scmd,  in scsih_qcmd()
5276 			ioc->put_smid_fast_path(ioc, smid, handle);  in scsih_qcmd()
5278 			ioc->put_smid_scsi_io(ioc, smid,  in scsih_qcmd()
5281 		ioc->put_smid_default(ioc, smid);  in scsih_qcmd()
5321 _scsih_scsi_ioc_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,  in _scsih_scsi_ioc_info()  argument
5332 	char *desc_scsi_state = ioc->tmp_string;  in _scsih_scsi_ioc_info()
5342 	if (ioc->hide_ir_msg)  in _scsih_scsi_ioc_info()
5466 		ioc_warn(ioc, "\t%s wwid(0x%016llx)\n",  in _scsih_scsi_ioc_info()
5469 		pcie_device = mpt3sas_get_pdev_from_target(ioc, priv_target);  in _scsih_scsi_ioc_info()
5471 			ioc_info(ioc, "\twwid(0x%016llx), port(%d)\n",  in _scsih_scsi_ioc_info()
5474 				ioc_info(ioc, "\tenclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_scsi_ioc_info()
5478 				ioc_info(ioc, "\tenclosure level(0x%04x), connector name( %s)\n",  in _scsih_scsi_ioc_info()
5484 		sas_device = mpt3sas_get_sdev_from_target(ioc, priv_target);  in _scsih_scsi_ioc_info()
5486 			ioc_warn(ioc, "\tsas_address(0x%016llx), phy(%d)\n",  in _scsih_scsi_ioc_info()
5489 			_scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_scsi_ioc_info()
5496 	ioc_warn(ioc, "\thandle(0x%04x), ioc_status(%s)(0x%04x), smid(%d)\n",  in _scsih_scsi_ioc_info()
5499 	ioc_warn(ioc, "\trequest_len(%d), underflow(%d), resid(%d)\n",  in _scsih_scsi_ioc_info()
5501 	ioc_warn(ioc, "\ttag(%d), transfer_count(%d), sc->result(0x%08x)\n",  in _scsih_scsi_ioc_info()
5504 	ioc_warn(ioc, "\tscsi_status(%s)(0x%02x), scsi_state(%s)(0x%02x)\n",  in _scsih_scsi_ioc_info()
5510 		ioc_warn(ioc, "\t[sense_key,asc,ascq]: [0x%02x,0x%02x,0x%02x], count(%d)\n",  in _scsih_scsi_ioc_info()
5517 		_scsih_response_code(ioc, response_bytes[0]);  in _scsih_scsi_ioc_info()
5528 _scsih_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_turn_on_pfa_led()  argument
5534 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_turn_on_pfa_led()
5545 	if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,  in _scsih_turn_on_pfa_led()
5547 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_turn_on_pfa_led()
5554 		dewtprintk(ioc,  in _scsih_turn_on_pfa_led()
5555 			   ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",  in _scsih_turn_on_pfa_led()
5571 _scsih_turn_off_pfa_led(struct MPT3SAS_ADAPTER *ioc,  in _scsih_turn_off_pfa_led()  argument
5585 	if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,  in _scsih_turn_off_pfa_led()
5587 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_turn_off_pfa_led()
5593 		dewtprintk(ioc,  in _scsih_turn_off_pfa_led()
5594 			   ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",  in _scsih_turn_off_pfa_led()
5608 _scsih_send_event_to_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_send_event_to_turn_on_pfa_led()  argument
5617 	fw_event->ioc = ioc;  in _scsih_send_event_to_turn_on_pfa_led()
5618 	_scsih_fw_event_add(ioc, fw_event);  in _scsih_send_event_to_turn_on_pfa_led()
5629 _scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_smart_predicted_fault()  argument
5640 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5641 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_smart_predicted_fault()
5654 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5656 	if (ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM)  in _scsih_smart_predicted_fault()
5657 		_scsih_send_event_to_turn_on_pfa_led(ioc, handle);  in _scsih_smart_predicted_fault()
5664 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_smart_predicted_fault()
5681 	mpt3sas_ctl_add_to_event_log(ioc, event_reply);  in _scsih_smart_predicted_fault()
5689 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5706 _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_io_done()  argument
5720 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_io_done()
5722 	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_io_done()
5728 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_io_done()
5756 		ioc->put_smid_scsi_io(ioc, smid,  in _scsih_io_done()
5767 		if ((!ioc->is_warpdrive &&  in _scsih_io_done()
5795 		const void *sense_data = mpt3sas_base_get_sense_buffer(ioc,  in _scsih_io_done()
5803 			_scsih_smart_predicted_fault(ioc,  in _scsih_io_done()
5805 		mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);  in _scsih_io_done()
5807 		if ((ioc->logging_level & MPT_DEBUG_REPLY) &&  in _scsih_io_done()
5811 			_scsih_scsi_ioc_info(ioc, scmd, mpi_reply, smid);  in _scsih_io_done()
5921 	if (scmd->result && (ioc->logging_level & MPT_DEBUG_REPLY))  in _scsih_io_done()
5922 		_scsih_scsi_ioc_info(ioc , scmd, mpi_reply, smid);  in _scsih_io_done()
5927 	mpt3sas_base_free_smid(ioc, smid);  in _scsih_io_done()
5940 _scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_update_vphys_after_reset()  argument
5958 	    &ioc->port_table_list, list) {  in _scsih_update_vphys_after_reset()
5971 	    (ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));  in _scsih_update_vphys_after_reset()
5974 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
5978 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_update_vphys_after_reset()
5987 	for (i = 0; i < ioc->sas_hba.num_phys; i++) {  in _scsih_update_vphys_after_reset()
6006 		if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,  in _scsih_update_vphys_after_reset()
6008 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
6021 		if (_scsih_get_sas_address(ioc, attached_handle,  in _scsih_update_vphys_after_reset()
6023 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
6035 		    port_next, &ioc->port_table_list, list) {  in _scsih_update_vphys_after_reset()
6068 				mport = mpt3sas_get_port_by_id(ioc, port_id, 1);  in _scsih_update_vphys_after_reset()
6075 					ioc_info(ioc,  in _scsih_update_vphys_after_reset()
6079 						&ioc->port_table_list);  in _scsih_update_vphys_after_reset()
6097 					    ioc, attached_sas_addr, port);  in _scsih_update_vphys_after_reset()
6138 _scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,  in _scsih_get_port_table_after_reset()  argument
6149 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys  in _scsih_get_port_table_after_reset()
6153 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_port_table_after_reset()
6158 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_get_port_table_after_reset()
6164 	for (i = 0; i < ioc->sas_hba.num_phys; i++) {  in _scsih_get_port_table_after_reset()
6172 		    ioc, attached_handle, &attached_sas_addr) != 0) {  in _scsih_get_port_table_after_reset()
6173 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_port_table_after_reset()
6222 _scsih_look_and_get_matched_port_entry(struct MPT3SAS_ADAPTER *ioc,  in _scsih_look_and_get_matched_port_entry()  argument
6231 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {  in _scsih_look_and_get_matched_port_entry()
6289 _scsih_del_phy_part_of_anther_port(struct MPT3SAS_ADAPTER *ioc,  in _scsih_del_phy_part_of_anther_port()  argument
6293 	struct _sas_node *sas_node = &ioc->sas_hba;  in _scsih_del_phy_part_of_anther_port()
6302 			    ioc, sas_node, &sas_node->phy[offset]);  in _scsih_del_phy_part_of_anther_port()
6322 _scsih_add_or_del_phys_from_existing_port(struct MPT3SAS_ADAPTER *ioc,  in _scsih_add_or_del_phys_from_existing_port()  argument
6327 	struct _sas_node *sas_node = &ioc->sas_hba;  in _scsih_add_or_del_phys_from_existing_port()
6331 	for (offset = 0; offset < ioc->sas_hba.num_phys; offset++) {  in _scsih_add_or_del_phys_from_existing_port()
6335 				    ioc, port_table, index, port_count,  in _scsih_add_or_del_phys_from_existing_port()
6341 				    ioc, sas_node, &sas_node->phy[offset]);  in _scsih_add_or_del_phys_from_existing_port()
6343 			    ioc, sas_node, &sas_node->phy[offset],  in _scsih_add_or_del_phys_from_existing_port()
6357 _scsih_del_dirty_vphy(struct MPT3SAS_ADAPTER *ioc)  in _scsih_del_dirty_vphy()  argument
6363 	    &ioc->port_table_list, list) {  in _scsih_del_dirty_vphy()
6369 				drsprintk(ioc, ioc_info(ioc,  in _scsih_del_dirty_vphy()
6390 _scsih_del_dirty_port_entries(struct MPT3SAS_ADAPTER *ioc)  in _scsih_del_dirty_port_entries()  argument
6395 	    &ioc->port_table_list, list) {  in _scsih_del_dirty_port_entries()
6400 		drsprintk(ioc, ioc_info(ioc,  in _scsih_del_dirty_port_entries()
6413 _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_port_refresh()  argument
6424 	drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6426 	    (unsigned long long)ioc->sas_hba.sas_address));  in _scsih_sas_port_refresh()
6428 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in _scsih_sas_port_refresh()
6430 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_port_refresh()
6435 	if (num_phys > ioc->sas_hba.nr_phys_allocated) {  in _scsih_sas_port_refresh()
6436 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_port_refresh()
6440 	ioc->sas_hba.num_phys = num_phys;  in _scsih_sas_port_refresh()
6442 	port_table = kcalloc(ioc->sas_hba.num_phys,  in _scsih_sas_port_refresh()
6447 	port_count = _scsih_get_port_table_after_reset(ioc, port_table);  in _scsih_sas_port_refresh()
6451 	drsprintk(ioc, ioc_info(ioc, "New Port table\n"));  in _scsih_sas_port_refresh()
6453 		drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6458 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list)  in _scsih_sas_port_refresh()
6461 	drsprintk(ioc, ioc_info(ioc, "Old Port table\n"));  in _scsih_sas_port_refresh()
6463 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {  in _scsih_sas_port_refresh()
6464 		drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6472 		ret = _scsih_look_and_get_matched_port_entry(ioc,  in _scsih_sas_port_refresh()
6475 			drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6485 			_scsih_add_or_del_phys_from_existing_port(ioc,  in _scsih_sas_port_refresh()
6498 				_scsih_add_or_del_phys_from_existing_port(ioc,  in _scsih_sas_port_refresh()
6523 _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)  in _scsih_alloc_vphy()  argument
6528 	port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_alloc_vphy()
6532 	vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy_num);  in _scsih_alloc_vphy()
6550 		ioc_info(ioc,  in _scsih_alloc_vphy()
6567 _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_host_refresh()  argument
6579 	dtmprintk(ioc,  in _scsih_sas_host_refresh()
6580 		  ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",  in _scsih_sas_host_refresh()
6581 			   (u64)ioc->sas_hba.sas_address));  in _scsih_sas_host_refresh()
6583 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys  in _scsih_sas_host_refresh()
6587 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6592 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6598 	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {  in _scsih_sas_host_refresh()
6601 			ioc->sas_hba.handle = le16_to_cpu(  in _scsih_sas_host_refresh()
6604 		if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {  in _scsih_sas_host_refresh()
6610 			ioc_info(ioc,  in _scsih_sas_host_refresh()
6613 			if (ioc->shost_recovery)  in _scsih_sas_host_refresh()
6615 			list_add_tail(&port->list, &ioc->port_table_list);  in _scsih_sas_host_refresh()
6623 			if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6625 				ioc_err(ioc,  in _scsih_sas_host_refresh()
6637 			if (!_scsih_alloc_vphy(ioc, port_id, i))  in _scsih_sas_host_refresh()
6639 			ioc->sas_hba.phy[i].hba_vphy = 1;  in _scsih_sas_host_refresh()
6646 		if (!ioc->sas_hba.phy[i].phy) {  in _scsih_sas_host_refresh()
6647 			if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6649 				ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6656 				ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6660 			ioc->sas_hba.phy[i].phy_id = i;  in _scsih_sas_host_refresh()
6661 			mpt3sas_transport_add_host_phy(ioc,  in _scsih_sas_host_refresh()
6662 				&ioc->sas_hba.phy[i], phy_pg0,  in _scsih_sas_host_refresh()
6663 				ioc->sas_hba.parent_dev);  in _scsih_sas_host_refresh()
6666 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;  in _scsih_sas_host_refresh()
6671 		ioc->sas_hba.phy[i].port =  in _scsih_sas_host_refresh()
6672 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_sas_host_refresh()
6673 		mpt3sas_transport_update_links(ioc, ioc->sas_hba.sas_address,  in _scsih_sas_host_refresh()
6675 		    ioc->sas_hba.phy[i].port);  in _scsih_sas_host_refresh()
6681 	for (i = ioc->sas_hba.num_phys;  in _scsih_sas_host_refresh()
6682 	     i < ioc->sas_hba.nr_phys_allocated; i++) {  in _scsih_sas_host_refresh()
6683 		if (ioc->sas_hba.phy[i].phy &&  in _scsih_sas_host_refresh()
6684 		    ioc->sas_hba.phy[i].phy->negotiated_linkrate >=  in _scsih_sas_host_refresh()
6686 			mpt3sas_transport_update_links(ioc,  in _scsih_sas_host_refresh()
6687 				ioc->sas_hba.sas_address, 0, i,  in _scsih_sas_host_refresh()
6701 _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_host_add()  argument
6716 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in _scsih_sas_host_add()
6718 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6723 	ioc->sas_hba.nr_phys_allocated = max_t(u8,  in _scsih_sas_host_add()
6725 	ioc->sas_hba.phy = kcalloc(ioc->sas_hba.nr_phys_allocated,  in _scsih_sas_host_add()
6727 	if (!ioc->sas_hba.phy) {  in _scsih_sas_host_add()
6728 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6732 	ioc->sas_hba.num_phys = num_phys;  in _scsih_sas_host_add()
6735 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *  in _scsih_sas_host_add()
6739 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6743 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_sas_host_add()
6745 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6752 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6758 	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *  in _scsih_sas_host_add()
6762 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6766 	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in _scsih_sas_host_add()
6768 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6775 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6780 	ioc->io_missing_delay =  in _scsih_sas_host_add()
6785 		ioc->device_missing_delay = (device_missing_delay &  in _scsih_sas_host_add()
6788 		ioc->device_missing_delay = device_missing_delay &  in _scsih_sas_host_add()
6791 	ioc->sas_hba.parent_dev = &ioc->shost->shost_gendev;  in _scsih_sas_host_add()
6792 	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {  in _scsih_sas_host_add()
6793 		if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,  in _scsih_sas_host_add()
6795 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6802 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6808 			ioc->sas_hba.handle = le16_to_cpu(sas_iounit_pg0->  in _scsih_sas_host_add()
6812 		if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {  in _scsih_sas_host_add()
6818 			ioc_info(ioc,  in _scsih_sas_host_add()
6822 			    &ioc->port_table_list);  in _scsih_sas_host_add()
6835 			if (!_scsih_alloc_vphy(ioc, port_id, i))  in _scsih_sas_host_add()
6837 			ioc->sas_hba.phy[i].hba_vphy = 1;  in _scsih_sas_host_add()
6840 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;  in _scsih_sas_host_add()
6841 		ioc->sas_hba.phy[i].phy_id = i;  in _scsih_sas_host_add()
6842 		ioc->sas_hba.phy[i].port =  in _scsih_sas_host_add()
6843 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_sas_host_add()
6844 		mpt3sas_transport_add_host_phy(ioc, &ioc->sas_hba.phy[i],  in _scsih_sas_host_add()
6845 		    phy_pg0, ioc->sas_hba.parent_dev);  in _scsih_sas_host_add()
6847 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_sas_host_add()
6848 	    MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, ioc->sas_hba.handle))) {  in _scsih_sas_host_add()
6849 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6853 	ioc->sas_hba.enclosure_handle =  in _scsih_sas_host_add()
6855 	ioc->sas_hba.sas_address = le64_to_cpu(sas_device_pg0.SASAddress);  in _scsih_sas_host_add()
6856 	ioc_info(ioc, "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n",  in _scsih_sas_host_add()
6857 		 ioc->sas_hba.handle,  in _scsih_sas_host_add()
6858 		 (u64)ioc->sas_hba.sas_address,  in _scsih_sas_host_add()
6859 		 ioc->sas_hba.num_phys);  in _scsih_sas_host_add()
6861 	if (ioc->sas_hba.enclosure_handle) {  in _scsih_sas_host_add()
6862 		if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_sas_host_add()
6864 		   ioc->sas_hba.enclosure_handle)))  in _scsih_sas_host_add()
6865 			ioc->sas_hba.enclosure_logical_id =  in _scsih_sas_host_add()
6884 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_expander_add()  argument
6904 	if (ioc->shost_recovery || ioc->pci_error_recovery)  in _scsih_expander_add()
6907 	if ((mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_expander_add()
6909 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6917 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6924 	if (_scsih_get_sas_address(ioc, parent_handle, &sas_address_parent)  in _scsih_expander_add()
6926 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6932 	if (sas_address_parent != ioc->sas_hba.sas_address) {  in _scsih_expander_add()
6933 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6934 		sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in _scsih_expander_add()
6936 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_expander_add()
6937 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6939 			rc = _scsih_expander_add(ioc, parent_handle);  in _scsih_expander_add()
6945 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6947 	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in _scsih_expander_add()
6948 	    sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_expander_add()
6949 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6957 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6966 	sas_expander->port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_expander_add()
6968 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6974 	ioc_info(ioc, "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n",  in _scsih_expander_add()
6985 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6992 	mpt3sas_port = mpt3sas_transport_port_add(ioc, handle,  in _scsih_expander_add()
6995 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7004 		if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,  in _scsih_expander_add()
7006 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7014 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_expander_add()
7016 		if ((mpt3sas_transport_add_expander_phy(ioc,  in _scsih_expander_add()
7019 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7028 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_expander_add()
7035 	_scsih_expander_node_add(ioc, sas_expander);  in _scsih_expander_add()
7041 		mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,  in _scsih_expander_add()
7054 mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,  in mpt3sas_expander_remove()  argument
7060 	if (ioc->shost_recovery)  in mpt3sas_expander_remove()
7066 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in mpt3sas_expander_remove()
7067 	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in mpt3sas_expander_remove()
7069 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in mpt3sas_expander_remove()
7071 		_scsih_expander_node_remove(ioc, sas_expander);  in mpt3sas_expander_remove()
7088 _scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_done()  argument
7092 	mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_done()
7093 	if (ioc->scsih_cmds.status == MPT3_CMD_NOT_USED)  in _scsih_done()
7095 	if (ioc->scsih_cmds.smid != smid)  in _scsih_done()
7097 	ioc->scsih_cmds.status |= MPT3_CMD_COMPLETE;  in _scsih_done()
7099 		memcpy(ioc->scsih_cmds.reply, mpi_reply,  in _scsih_done()
7101 		ioc->scsih_cmds.status |= MPT3_CMD_REPLY_VALID;  in _scsih_done()
7103 	ioc->scsih_cmds.status &= ~MPT3_CMD_PENDING;  in _scsih_done()
7104 	complete(&ioc->scsih_cmds.done);  in _scsih_done()
7124 _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,  in _scsih_check_access_status()  argument
7172 	ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",  in _scsih_check_access_status()
7186 _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_device()  argument
7201 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_check_device()
7220 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7222 	port = mpt3sas_get_port_by_id(ioc, sas_device_pg0.PhysicalPort, 0);  in _scsih_check_device()
7225 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_check_device()
7254 		enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_check_device()
7271 		ioc_err(ioc, "device is not present handle(0x%04x), flags!!!\n",  in _scsih_check_device()
7277 	if (_scsih_check_access_status(ioc, sas_address, handle,  in _scsih_check_device()
7281 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7282 	_scsih_ublock_io_device(ioc, sas_address, port);  in _scsih_check_device()
7289 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7306 _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,  in _scsih_add_device()  argument
7318 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_add_device()
7320 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7328 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7337 	set_bit(handle, ioc->pend_os_device_add);  in _scsih_add_device()
7343 		ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",  in _scsih_add_device()
7349 	if (_scsih_check_access_status(ioc, sas_address, handle,  in _scsih_add_device()
7354 	sas_device = mpt3sas_get_sdev_by_addr(ioc,  in _scsih_add_device()
7355 	    sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_add_device()
7357 		clear_bit(handle, ioc->pend_os_device_add);  in _scsih_add_device()
7364 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_add_device()
7367 			ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",  in _scsih_add_device()
7374 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7381 	if (_scsih_get_sas_address(ioc,  in _scsih_add_device()
7384 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7396 	sas_device->port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_add_device()
7398 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7430 	ioc_info(ioc,  in _scsih_add_device()
7434 	if (ioc->wait_for_discovery_to_complete)  in _scsih_add_device()
7435 		_scsih_sas_device_init_add(ioc, sas_device);  in _scsih_add_device()
7437 		_scsih_sas_device_add(ioc, sas_device);  in _scsih_add_device()
7450 _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_remove_device()  argument
7455 	if ((ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM) &&  in _scsih_remove_device()
7457 		_scsih_turn_off_pfa_led(ioc, sas_device);  in _scsih_remove_device()
7461 	dewtprintk(ioc,  in _scsih_remove_device()
7462 		   ioc_info(ioc, "%s: enter: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7466 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_remove_device()
7472 		_scsih_ublock_io_device(ioc, sas_device->sas_address,  in _scsih_remove_device()
7478 	if (!ioc->hide_drives)  in _scsih_remove_device()
7479 		mpt3sas_transport_port_remove(ioc,  in _scsih_remove_device()
7484 	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7487 	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);  in _scsih_remove_device()
7489 	dewtprintk(ioc,  in _scsih_remove_device()
7490 		   ioc_info(ioc, "%s: exit: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7493 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_remove_device()
7504 _scsih_sas_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_topology_change_event_debug()  argument
7532 	ioc_info(ioc, "sas topology change: (%s)\n", status_str);  in _scsih_sas_topology_change_event_debug()
7582 _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_topology_change_event()  argument
7598 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_topology_change_event()
7599 		_scsih_sas_topology_change_event_debug(ioc, event_data);  in _scsih_sas_topology_change_event()
7601 	if (ioc->shost_recovery || ioc->remove_host || ioc->pci_error_recovery)  in _scsih_sas_topology_change_event()
7604 	if (!ioc->sas_hba.num_phys)  in _scsih_sas_topology_change_event()
7605 		_scsih_sas_host_add(ioc);  in _scsih_sas_topology_change_event()
7607 		_scsih_sas_host_refresh(ioc);  in _scsih_sas_topology_change_event()
7610 		dewtprintk(ioc, ioc_info(ioc, "ignoring expander event\n"));  in _scsih_sas_topology_change_event()
7615 	port = mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0);  in _scsih_sas_topology_change_event()
7619 		if (_scsih_expander_add(ioc, parent_handle) != 0)  in _scsih_sas_topology_change_event()
7622 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7623 	sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,  in _scsih_sas_topology_change_event()
7629 	} else if (parent_handle < ioc->sas_hba.num_phys) {  in _scsih_sas_topology_change_event()
7630 		sas_address = ioc->sas_hba.sas_address;  in _scsih_sas_topology_change_event()
7631 		max_phys = ioc->sas_hba.num_phys;  in _scsih_sas_topology_change_event()
7633 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7636 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7641 			dewtprintk(ioc,  in _scsih_sas_topology_change_event()
7642 				   ioc_info(ioc, "ignoring expander event\n"));  in _scsih_sas_topology_change_event()
7645 		if (ioc->remove_host || ioc->pci_error_recovery)  in _scsih_sas_topology_change_event()
7664 			if (ioc->shost_recovery)  in _scsih_sas_topology_change_event()
7670 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_sas_topology_change_event()
7676 			_scsih_check_device(ioc, sas_address, handle,  in _scsih_sas_topology_change_event()
7679 			if (!test_bit(handle, ioc->pend_os_device_add))  in _scsih_sas_topology_change_event()
7686 			if (ioc->shost_recovery)  in _scsih_sas_topology_change_event()
7689 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_sas_topology_change_event()
7692 			_scsih_add_device(ioc, handle, phy_number, 0);  in _scsih_sas_topology_change_event()
7697 			_scsih_device_remove_by_handle(ioc, handle);  in _scsih_sas_topology_change_event()
7705 		mpt3sas_expander_remove(ioc, sas_address, port);  in _scsih_sas_topology_change_event()
7717 _scsih_sas_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_status_change_event_debug()  argument
7766 	ioc_info(ioc, "device status change: (%s)\thandle(0x%04x), sas address(0x%016llx), tag(%d)",  in _scsih_sas_device_status_change_event_debug()
7783 _scsih_sas_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_status_change_event()  argument
7794 	if ((ioc->facts.HeaderVersion >> 8) < 0xC)  in _scsih_sas_device_status_change_event()
7803 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_status_change_event()
7805 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_sas_device_status_change_event()
7807 	    mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0));  in _scsih_sas_device_status_change_event()
7822 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_device_status_change_event()
7823 		ioc_info(ioc,  in _scsih_sas_device_status_change_event()
7832 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_status_change_event()
7846 _scsih_check_pcie_access_status(struct MPT3SAS_ADAPTER *ioc, u64 wwid,  in _scsih_check_pcie_access_status()  argument
7862 		ioc_info(ioc,  in _scsih_check_pcie_access_status()
7911 		ioc_err(ioc, "NVMe discovery error(0x%02x): wwid(0x%016llx), handle(0x%04x)\n",  in _scsih_check_pcie_access_status()
7919 	ioc_info(ioc, "NVMe discovery error(%s): wwid(0x%016llx), handle(0x%04x)\n",  in _scsih_check_pcie_access_status()
7931 _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_remove_from_sml()  argument
7936 	dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7937 		   ioc_info(ioc, "%s: enter: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7941 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7942 			   ioc_info(ioc, "%s: enter: enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7947 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7948 			   ioc_info(ioc, "%s: enter: enclosure level(0x%04x), connector name(%s)\n",  in _scsih_pcie_device_remove_from_sml()
7956 		_scsih_ublock_io_device(ioc, pcie_device->wwid, NULL);  in _scsih_pcie_device_remove_from_sml()
7960 	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7963 		ioc_info(ioc, "removing : enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7967 		ioc_info(ioc, "removing: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove_from_sml()
7974 	dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7975 		   ioc_info(ioc, "%s: exit: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7979 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7980 			   ioc_info(ioc, "%s: exit: enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7985 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7986 			   ioc_info(ioc, "%s: exit: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove_from_sml()
8001 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_check_device()  argument
8013 	if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_pcie_check_device()
8027 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8028 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_check_device()
8031 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8060 		ioc_info(ioc, "device is not present handle(0x%04x), flags!!!\n",  in _scsih_pcie_check_device()
8062 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8068 	if (_scsih_check_pcie_access_status(ioc, wwid, handle,  in _scsih_pcie_check_device()
8070 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8075 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8078 	_scsih_ublock_io_device(ioc, wwid, NULL);  in _scsih_pcie_check_device()
8093 _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_add_device()  argument
8103 	if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_pcie_add_device()
8105 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8112 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8117 	set_bit(handle, ioc->pend_os_device_add);  in _scsih_pcie_add_device()
8123 		ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",  in _scsih_pcie_add_device()
8129 	if (_scsih_check_pcie_access_status(ioc, wwid, handle,  in _scsih_pcie_add_device()
8137 	pcie_device = mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_add_device()
8139 		clear_bit(handle, ioc->pend_os_device_add);  in _scsih_pcie_add_device()
8150 		if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,  in _scsih_pcie_add_device()
8153 			ioc_err(ioc,  in _scsih_pcie_add_device()
8162 			ioc_err(ioc,  in _scsih_pcie_add_device()
8171 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8177 	pcie_device->id = ioc->pcie_target_id++;  in _scsih_pcie_add_device()
8205 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_pcie_add_device()
8223 		if (pcie_device->shutdown_latency > ioc->max_shutdown_latency)  in _scsih_pcie_add_device()
8224 			ioc->max_shutdown_latency =  in _scsih_pcie_add_device()
8234 	if (ioc->wait_for_discovery_to_complete)  in _scsih_pcie_add_device()
8235 		_scsih_pcie_device_init_add(ioc, pcie_device);  in _scsih_pcie_add_device()
8237 		_scsih_pcie_device_add(ioc, pcie_device);  in _scsih_pcie_add_device()
8251 _scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_topology_change_event_debug()  argument
8279 	ioc_info(ioc, "pcie topology change: (%s)\n", status_str);  in _scsih_pcie_topology_change_event_debug()
8331 _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_topology_change_event()  argument
8344 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_pcie_topology_change_event()
8345 		_scsih_pcie_topology_change_event_debug(ioc, event_data);  in _scsih_pcie_topology_change_event()
8347 	if (ioc->shost_recovery || ioc->remove_host ||  in _scsih_pcie_topology_change_event()
8348 		ioc->pci_error_recovery)  in _scsih_pcie_topology_change_event()
8352 		dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));  in _scsih_pcie_topology_change_event()
8359 			dewtprintk(ioc,  in _scsih_pcie_topology_change_event()
8360 				   ioc_info(ioc, "ignoring switch event\n"));  in _scsih_pcie_topology_change_event()
8363 		if (ioc->remove_host || ioc->pci_error_recovery)  in _scsih_pcie_topology_change_event()
8378 			if (ioc->shost_recovery)  in _scsih_pcie_topology_change_event()
8385 			_scsih_pcie_check_device(ioc, handle);  in _scsih_pcie_topology_change_event()
8393 			spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_topology_change_event()
8394 			pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_pcie_topology_change_event()
8395 			spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_topology_change_event()
8402 			if (!test_bit(handle, ioc->pend_os_device_add))  in _scsih_pcie_topology_change_event()
8405 			dewtprintk(ioc,  in _scsih_pcie_topology_change_event()
8406 				   ioc_info(ioc, "handle(0x%04x) device not found: convert event to a device add\n",  in _scsih_pcie_topology_change_event()
8413 			if (ioc->shost_recovery)  in _scsih_pcie_topology_change_event()
8418 			rc = _scsih_pcie_add_device(ioc, handle);  in _scsih_pcie_topology_change_event()
8430 			_scsih_pcie_device_remove_by_handle(ioc, handle);  in _scsih_pcie_topology_change_event()
8443 _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_status_change_event_debug()  argument
8490 	ioc_info(ioc, "PCIE device status change: (%s)\n"  in _scsih_pcie_device_status_change_event_debug()
8509 _scsih_pcie_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_status_change_event()  argument
8518 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_pcie_device_status_change_event()
8519 		_scsih_pcie_device_status_change_event_debug(ioc,  in _scsih_pcie_device_status_change_event()
8528 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_status_change_event()
8530 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_device_status_change_event()
8548 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_status_change_event()
8559 _scsih_sas_enclosure_dev_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_enclosure_dev_status_change_event_debug()  argument
8576 	ioc_info(ioc, "enclosure status change: (%s)\n"  in _scsih_sas_enclosure_dev_status_change_event_debug()
8591 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_enclosure_dev_status_change_event()  argument
8601 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_enclosure_dev_status_change_event()
8602 		_scsih_sas_enclosure_dev_status_change_event_debug(ioc,  in _scsih_sas_enclosure_dev_status_change_event()
8605 	if (ioc->shost_recovery)  in _scsih_sas_enclosure_dev_status_change_event()
8610 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_sas_enclosure_dev_status_change_event()
8619 				ioc_info(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_enclosure_dev_status_change_event()
8623 			rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_sas_enclosure_dev_status_change_event()
8635 							&ioc->enclosure_list);  in _scsih_sas_enclosure_dev_status_change_event()
8656 _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_broadcast_primitive_event()  argument
8677 	mutex_lock(&ioc->tm_cmds.mutex);  in _scsih_sas_broadcast_primitive_event()
8678 	ioc_info(ioc, "%s: enter: phy number(%d), width(%d)\n",  in _scsih_sas_broadcast_primitive_event()
8681 	_scsih_block_io_all_device(ioc);  in _scsih_sas_broadcast_primitive_event()
8683 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8684 	mpi_reply = ioc->tm_cmds.reply;  in _scsih_sas_broadcast_primitive_event()
8689 		dewtprintk(ioc, ioc_info(ioc, "%s: giving up\n", __func__));  in _scsih_sas_broadcast_primitive_event()
8692 		dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8693 			   ioc_info(ioc, "%s: %d retry\n",  in _scsih_sas_broadcast_primitive_event()
8698 	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {  in _scsih_sas_broadcast_primitive_event()
8699 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8701 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_sas_broadcast_primitive_event()
8726 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8729 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8730 		r = mpt3sas_scsih_issue_tm(ioc, handle, 0, 0, lun,  in _scsih_sas_broadcast_primitive_event()
8737 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8746 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8755 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8761 			dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8762 				   ioc_info(ioc, "%s: ABORT_TASK: giving up\n",  in _scsih_sas_broadcast_primitive_event()
8764 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8768 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8771 		r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->channel, sdev->id,  in _scsih_sas_broadcast_primitive_event()
8788 		spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8791 	if (ioc->broadcast_aen_pending) {  in _scsih_sas_broadcast_primitive_event()
8792 		dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8793 			   ioc_info(ioc,  in _scsih_sas_broadcast_primitive_event()
8796 		 ioc->broadcast_aen_pending = 0;  in _scsih_sas_broadcast_primitive_event()
8801 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8804 	dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8805 		   ioc_info(ioc, "%s - exit, query_count = %d termination_count = %d\n",  in _scsih_sas_broadcast_primitive_event()
8808 	ioc->broadcast_aen_busy = 0;  in _scsih_sas_broadcast_primitive_event()
8809 	if (!ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8810 		_scsih_ublock_io_all_device(ioc);  in _scsih_sas_broadcast_primitive_event()
8811 	mutex_unlock(&ioc->tm_cmds.mutex);  in _scsih_sas_broadcast_primitive_event()
8821 _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_discovery_event()  argument
8827 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {  in _scsih_sas_discovery_event()
8828 		ioc_info(ioc, "discovery event: (%s)",  in _scsih_sas_discovery_event()
8838 	    !ioc->sas_hba.num_phys) {  in _scsih_sas_discovery_event()
8839 		if (disable_discovery > 0 && ioc->shost_recovery) {  in _scsih_sas_discovery_event()
8841 			while (ioc->shost_recovery)  in _scsih_sas_discovery_event()
8844 		_scsih_sas_host_add(ioc);  in _scsih_sas_discovery_event()
8856 _scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_discovery_error_event()  argument
8864 …ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_po…  in _scsih_sas_device_discovery_error_event()
8870 …ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_po…  in _scsih_sas_device_discovery_error_event()
8887 _scsih_pcie_enumeration_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_enumeration_event()  argument
8893 	if (!(ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK))  in _scsih_pcie_enumeration_event()
8896 	ioc_info(ioc, "pcie enumeration event: (%s) Flag 0x%02x",  in _scsih_pcie_enumeration_event()
8915 _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)  in _scsih_ir_fastpath()  argument
8925 	if (ioc->hba_mpi_version_belonged == MPI2_VERSION)  in _scsih_ir_fastpath()
8928 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_ir_fastpath()
8930 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_ir_fastpath()
8931 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_ir_fastpath()
8935 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_ir_fastpath()
8937 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_ir_fastpath()
8939 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _scsih_ir_fastpath()
8940 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_fastpath()
8945 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_ir_fastpath()
8946 	ioc->scsih_cmds.smid = smid;  in _scsih_ir_fastpath()
8953 	dewtprintk(ioc,  in _scsih_ir_fastpath()
8954 … ioc_info(ioc, "IR RAID_ACTION: turning fast path on for handle(0x%04x), phys_disk_num (0x%02x)\n",  in _scsih_ir_fastpath()
8957 	init_completion(&ioc->scsih_cmds.done);  in _scsih_ir_fastpath()
8958 	ioc->put_smid_default(ioc, smid);  in _scsih_ir_fastpath()
8959 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);  in _scsih_ir_fastpath()
8961 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_ir_fastpath()
8962 		mpt3sas_check_cmd_timeout(ioc,  in _scsih_ir_fastpath()
8963 		    ioc->scsih_cmds.status, mpi_request,  in _scsih_ir_fastpath()
8969 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_ir_fastpath()
8971 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_ir_fastpath()
8979 			dewtprintk(ioc,  in _scsih_ir_fastpath()
8980 				   ioc_info(ioc, "IR RAID_ACTION: failed: ioc_status(0x%04x), loginfo(0x%08x)!!!\n",  in _scsih_ir_fastpath()
8984 			dewtprintk(ioc,  in _scsih_ir_fastpath()
8985 				   ioc_info(ioc, "IR RAID_ACTION: completed successfully\n"));  in _scsih_ir_fastpath()
8989 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_fastpath()
8990 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_ir_fastpath()
8993 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _scsih_ir_fastpath()
9019 _scsih_sas_volume_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_volume_add()  argument
9028 	mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);  in _scsih_sas_volume_add()
9030 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_volume_add()
9035 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9036 	raid_device = _scsih_raid_device_find_by_wwid(ioc, wwid);  in _scsih_sas_volume_add()
9037 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9044 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_volume_add()
9049 	raid_device->id = ioc->sas_id++;  in _scsih_sas_volume_add()
9053 	_scsih_raid_device_add(ioc, raid_device);  in _scsih_sas_volume_add()
9054 	if (!ioc->wait_for_discovery_to_complete) {  in _scsih_sas_volume_add()
9055 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_sas_volume_add()
9058 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_sas_volume_add()
9060 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9061 		_scsih_determine_boot_device(ioc, raid_device, 1);  in _scsih_sas_volume_add()
9062 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9073 _scsih_sas_volume_delete(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_sas_volume_delete()  argument
9080 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_delete()
9081 	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_volume_delete()
9088 		ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_sas_volume_delete()
9093 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_delete()
9105 _scsih_sas_pd_expose(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_expose()  argument
9114 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_expose()
9115 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_expose()
9119 		clear_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_expose()
9127 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_expose()
9145 _scsih_sas_pd_hide(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_hide()  argument
9156 	mpt3sas_config_get_volume_handle(ioc, handle, &volume_handle);  in _scsih_sas_pd_hide()
9158 		mpt3sas_config_get_volume_wwid(ioc, volume_handle,  in _scsih_sas_pd_hide()
9161 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_hide()
9162 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_hide()
9164 		set_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_hide()
9174 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_hide()
9179 	_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_hide()
9194 _scsih_sas_pd_delete(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_delete()  argument
9199 	_scsih_device_remove_by_handle(ioc, handle);  in _scsih_sas_pd_delete()
9209 _scsih_sas_pd_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_add()  argument
9220 	set_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_add()
9222 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_add()
9224 		_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_add()
9229 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_sas_pd_add()
9231 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_pd_add()
9239 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_pd_add()
9245 	if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))  in _scsih_sas_pd_add()
9246 		mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_sas_pd_add()
9248 		    mpt3sas_get_port_by_id(ioc,  in _scsih_sas_pd_add()
9251 	_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_add()
9252 	_scsih_add_device(ioc, handle, 0, 1);  in _scsih_sas_pd_add()
9262 _scsih_sas_ir_config_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_config_change_event_debug()  argument
9272 	ioc_info(ioc, "raid config change: (%s), elements(%d)\n",  in _scsih_sas_ir_config_change_event_debug()
9340 _scsih_sas_ir_config_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_config_change_event()  argument
9350 	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&  in _scsih_sas_ir_config_change_event()
9351 	     (!ioc->hide_ir_msg))  in _scsih_sas_ir_config_change_event()
9352 		_scsih_sas_ir_config_change_event_debug(ioc, event_data);  in _scsih_sas_ir_config_change_event()
9358 	if (ioc->shost_recovery &&  in _scsih_sas_ir_config_change_event()
9359 	    ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _scsih_sas_ir_config_change_event()
9362 				_scsih_ir_fastpath(ioc,  in _scsih_sas_ir_config_change_event()
9375 				_scsih_sas_volume_add(ioc, element);  in _scsih_sas_ir_config_change_event()
9380 				_scsih_sas_volume_delete(ioc,  in _scsih_sas_ir_config_change_event()
9384 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9385 				_scsih_sas_pd_hide(ioc, element);  in _scsih_sas_ir_config_change_event()
9388 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9389 				_scsih_sas_pd_expose(ioc, element);  in _scsih_sas_ir_config_change_event()
9392 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9393 				_scsih_sas_pd_add(ioc, element);  in _scsih_sas_ir_config_change_event()
9396 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9397 				_scsih_sas_pd_delete(ioc, element);  in _scsih_sas_ir_config_change_event()
9410 _scsih_sas_ir_volume_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_volume_event()  argument
9422 	if (ioc->shost_recovery)  in _scsih_sas_ir_volume_event()
9430 	if (!ioc->hide_ir_msg)  in _scsih_sas_ir_volume_event()
9431 		dewtprintk(ioc,  in _scsih_sas_ir_volume_event()
9432 			   ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",  in _scsih_sas_ir_volume_event()
9439 		_scsih_sas_volume_delete(ioc, handle);  in _scsih_sas_ir_volume_event()
9446 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_volume_event()
9447 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_ir_volume_event()
9448 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_volume_event()
9453 		mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);  in _scsih_sas_ir_volume_event()
9455 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_volume_event()
9462 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_volume_event()
9467 		raid_device->id = ioc->sas_id++;  in _scsih_sas_ir_volume_event()
9471 		_scsih_raid_device_add(ioc, raid_device);  in _scsih_sas_ir_volume_event()
9472 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_sas_ir_volume_event()
9475 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_sas_ir_volume_event()
9491 _scsih_sas_ir_physical_disk_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_physical_disk_event()  argument
9504 	if (ioc->shost_recovery)  in _scsih_sas_ir_physical_disk_event()
9513 	if (!ioc->hide_ir_msg)  in _scsih_sas_ir_physical_disk_event()
9514 		dewtprintk(ioc,  in _scsih_sas_ir_physical_disk_event()
9515 			   ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",  in _scsih_sas_ir_physical_disk_event()
9527 		if (!ioc->is_warpdrive)  in _scsih_sas_ir_physical_disk_event()
9528 			set_bit(handle, ioc->pd_handles);  in _scsih_sas_ir_physical_disk_event()
9530 		sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_ir_physical_disk_event()
9536 		if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_sas_ir_physical_disk_event()
9539 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_physical_disk_event()
9547 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_physical_disk_event()
9553 		if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))  in _scsih_sas_ir_physical_disk_event()
9554 			mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_sas_ir_physical_disk_event()
9556 			    mpt3sas_get_port_by_id(ioc,  in _scsih_sas_ir_physical_disk_event()
9559 		_scsih_add_device(ioc, handle, 0, 1);  in _scsih_sas_ir_physical_disk_event()
9578 _scsih_sas_ir_operation_status_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_operation_status_event_debug()  argument
9604 	ioc_info(ioc, "raid operational status: (%s)\thandle(0x%04x), percent complete(%d)\n",  in _scsih_sas_ir_operation_status_event_debug()
9617 _scsih_sas_ir_operation_status_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_operation_status_event()  argument
9627 	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&  in _scsih_sas_ir_operation_status_event()
9628 	    (!ioc->hide_ir_msg))  in _scsih_sas_ir_operation_status_event()
9629 		_scsih_sas_ir_operation_status_event_debug(ioc,  in _scsih_sas_ir_operation_status_event()
9635 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_operation_status_event()
9637 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_ir_operation_status_event()
9641 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_operation_status_event()
9653 _scsih_prep_device_scan(struct MPT3SAS_ADAPTER *ioc)  in _scsih_prep_device_scan()  argument
9658 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_prep_device_scan()
9671 _scsih_update_device_qdepth(struct MPT3SAS_ADAPTER *ioc)  in _scsih_update_device_qdepth()  argument
9679 	ioc_info(ioc, "Update devices with firmware reported queue depth\n");  in _scsih_update_device_qdepth()
9680 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_update_device_qdepth()
9686 				qdepth = ioc->max_nvme_qd;  in _scsih_update_device_qdepth()
9690 				    ioc->max_wideport_qd : ioc->max_narrowport_qd;  in _scsih_update_device_qdepth()
9693 				qdepth = ioc->max_sata_qd;  in _scsih_update_device_qdepth()
9710 _scsih_mark_responding_sas_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_sas_device()  argument
9719 	    ioc, sas_device_pg0->PhysicalPort, 0);  in _scsih_mark_responding_sas_device()
9723 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_mark_responding_sas_device()
9726 			ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",  in _scsih_mark_responding_sas_device()
9729 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_mark_responding_sas_device()
9730 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in _scsih_mark_responding_sas_device()
9798 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_mark_responding_sas_device()
9807 _scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_create_enclosure_list_after_reset()  argument
9815 	mpt3sas_free_enclosure_list(ioc);  in _scsih_create_enclosure_list_after_reset()
9823 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_create_enclosure_list_after_reset()
9827 		rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_create_enclosure_list_after_reset()
9838 						&ioc->enclosure_list);  in _scsih_create_enclosure_list_after_reset()
9852 _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_sas_devices()  argument
9860 	ioc_info(ioc, "search for end-devices: start\n");  in _scsih_search_responding_sas_devices()
9862 	if (list_empty(&ioc->sas_device_list))  in _scsih_search_responding_sas_devices()
9866 	while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_search_responding_sas_devices()
9877 		_scsih_mark_responding_sas_device(ioc, &sas_device_pg0);  in _scsih_search_responding_sas_devices()
9881 	ioc_info(ioc, "search for end-devices: complete\n");  in _scsih_search_responding_sas_devices()
9893 _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_pcie_device()  argument
9901 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_mark_responding_pcie_device()
9902 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {  in _scsih_mark_responding_pcie_device()
9932 			    (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {  in _scsih_mark_responding_pcie_device()
9957 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_mark_responding_pcie_device()
9968 _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_pcie_devices()  argument
9976 	ioc_info(ioc, "search for end-devices: start\n");  in _scsih_search_responding_pcie_devices()
9978 	if (list_empty(&ioc->pcie_device_list))  in _scsih_search_responding_pcie_devices()
9982 	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_search_responding_pcie_devices()
9988 			ioc_info(ioc, "\tbreak from %s: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_search_responding_pcie_devices()
9997 		_scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);  in _scsih_search_responding_pcie_devices()
10000 	ioc_info(ioc, "search for PCIe end-devices: complete\n");  in _scsih_search_responding_pcie_devices()
10013 _scsih_mark_responding_raid_device(struct MPT3SAS_ADAPTER *ioc, u64 wwid,  in _scsih_mark_responding_raid_device()  argument
10021 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10022 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_mark_responding_raid_device()
10031 			spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10041 			mpt3sas_init_warpdrive_properties(ioc, raid_device);  in _scsih_mark_responding_raid_device()
10042 			spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10044 				spin_unlock_irqrestore(&ioc->raid_device_lock,  in _scsih_mark_responding_raid_device()
10053 			spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10057 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10068 _scsih_search_responding_raid_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_raid_devices()  argument
10078 	if (!ioc->ir_firmware)  in _scsih_search_responding_raid_devices()
10081 	ioc_info(ioc, "search for raid volumes: start\n");  in _scsih_search_responding_raid_devices()
10083 	if (list_empty(&ioc->raid_device_list))  in _scsih_search_responding_raid_devices()
10087 	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10095 		if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10103 			_scsih_mark_responding_raid_device(ioc,  in _scsih_search_responding_raid_devices()
10108 	if (!ioc->is_warpdrive) {  in _scsih_search_responding_raid_devices()
10110 		memset(ioc->pd_handles, 0, ioc->pd_handles_sz);  in _scsih_search_responding_raid_devices()
10111 		while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10120 			set_bit(handle, ioc->pd_handles);  in _scsih_search_responding_raid_devices()
10124 	ioc_info(ioc, "search for responding raid volumes: complete\n");  in _scsih_search_responding_raid_devices()
10136 _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_expander()  argument
10147 	    ioc, expander_pg0->PhysicalPort, 0);  in _scsih_mark_responding_expander()
10151 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_mark_responding_expander()
10154 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_mark_responding_expander()
10155 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in _scsih_mark_responding_expander()
10181 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_mark_responding_expander()
10192 _scsih_search_responding_expanders(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_expanders()  argument
10201 	ioc_info(ioc, "search for expanders: start\n");  in _scsih_search_responding_expanders()
10203 	if (list_empty(&ioc->sas_expander_list))  in _scsih_search_responding_expanders()
10207 	while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_search_responding_expanders()
10221 		    (ioc->multipath_on_hba ?  in _scsih_search_responding_expanders()
10223 		_scsih_mark_responding_expander(ioc, &expander_pg0);  in _scsih_search_responding_expanders()
10227 	ioc_info(ioc, "search for expanders: complete\n");  in _scsih_search_responding_expanders()
10235 _scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_remove_unresponding_devices()  argument
10245 	ioc_info(ioc, "removing unresponding devices: start\n");  in _scsih_remove_unresponding_devices()
10248 	ioc_info(ioc, "removing unresponding devices: end-devices\n");  in _scsih_remove_unresponding_devices()
10253 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_remove_unresponding_devices()
10260 	    &ioc->sas_device_init_list, list) {  in _scsih_remove_unresponding_devices()
10266 	    &ioc->sas_device_list, list) {  in _scsih_remove_unresponding_devices()
10272 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_remove_unresponding_devices()
10278 		_scsih_remove_device(ioc, sas_device);  in _scsih_remove_unresponding_devices()
10283 	ioc_info(ioc, "Removing unresponding devices: pcie end-devices\n");  in _scsih_remove_unresponding_devices()
10285 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_remove_unresponding_devices()
10291 	    &ioc->pcie_device_init_list, list) {  in _scsih_remove_unresponding_devices()
10297 	    &ioc->pcie_device_list, list) {  in _scsih_remove_unresponding_devices()
10303 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_remove_unresponding_devices()
10306 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in _scsih_remove_unresponding_devices()
10312 	if (ioc->ir_firmware) {  in _scsih_remove_unresponding_devices()
10313 		ioc_info(ioc, "removing unresponding devices: volumes\n");  in _scsih_remove_unresponding_devices()
10315 		    &ioc->raid_device_list, list) {  in _scsih_remove_unresponding_devices()
10317 				_scsih_sas_volume_delete(ioc,  in _scsih_remove_unresponding_devices()
10325 	ioc_info(ioc, "removing unresponding devices: expanders\n");  in _scsih_remove_unresponding_devices()
10326 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_remove_unresponding_devices()
10329 	    &ioc->sas_expander_list, list) {  in _scsih_remove_unresponding_devices()
10335 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_remove_unresponding_devices()
10338 		_scsih_expander_node_remove(ioc, sas_expander);  in _scsih_remove_unresponding_devices()
10341 	ioc_info(ioc, "removing unresponding devices: complete\n");  in _scsih_remove_unresponding_devices()
10344 	_scsih_ublock_io_all_device(ioc);  in _scsih_remove_unresponding_devices()
10348 _scsih_refresh_expander_links(struct MPT3SAS_ADAPTER *ioc,  in _scsih_refresh_expander_links()  argument
10356 		if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,  in _scsih_refresh_expander_links()
10358 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_refresh_expander_links()
10363 		mpt3sas_transport_update_links(ioc, sas_expander->sas_address,  in _scsih_refresh_expander_links()
10375 _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_scan_for_devices_after_reset()  argument
10406 	ioc_info(ioc, "scan devices: start\n");  in _scsih_scan_for_devices_after_reset()
10408 	_scsih_sas_host_refresh(ioc);  in _scsih_scan_for_devices_after_reset()
10410 	ioc_info(ioc, "\tscan devices: expanders start\n");  in _scsih_scan_for_devices_after_reset()
10414 	while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_scan_for_devices_after_reset()
10419 			ioc_info(ioc, "\tbreak from expander scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10424 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_scan_for_devices_after_reset()
10427 		    ioc, le64_to_cpu(expander_pg0.SASAddress),  in _scsih_scan_for_devices_after_reset()
10428 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10429 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_scan_for_devices_after_reset()
10431 			_scsih_refresh_expander_links(ioc, expander_device,  in _scsih_scan_for_devices_after_reset()
10434 			ioc_info(ioc, "\tBEFORE adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10437 			_scsih_expander_add(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10438 			ioc_info(ioc, "\tAFTER adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10444 	ioc_info(ioc, "\tscan devices: expanders complete\n");  in _scsih_scan_for_devices_after_reset()
10446 	if (!ioc->ir_firmware)  in _scsih_scan_for_devices_after_reset()
10449 	ioc_info(ioc, "\tscan devices: phys disk start\n");  in _scsih_scan_for_devices_after_reset()
10453 	while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10459 			ioc_info(ioc, "\tbreak from phys disk scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10465 		sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10470 		if (mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10477 			ioc_info(ioc, "\tbreak from phys disk scan ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10482 		if (!_scsih_get_sas_address(ioc, parent_handle,  in _scsih_scan_for_devices_after_reset()
10484 			ioc_info(ioc, "\tBEFORE adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10488 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_scan_for_devices_after_reset()
10491 			    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10492 			set_bit(handle, ioc->pd_handles);  in _scsih_scan_for_devices_after_reset()
10498 			while (_scsih_add_device(ioc, handle, retry_count++,  in _scsih_scan_for_devices_after_reset()
10502 			ioc_info(ioc, "\tAFTER adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10508 	ioc_info(ioc, "\tscan devices: phys disk complete\n");  in _scsih_scan_for_devices_after_reset()
10510 	ioc_info(ioc, "\tscan devices: volumes start\n");  in _scsih_scan_for_devices_after_reset()
10514 	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10519 			ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10524 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_scan_for_devices_after_reset()
10525 		raid_device = _scsih_raid_device_find_by_wwid(ioc,  in _scsih_scan_for_devices_after_reset()
10527 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_scan_for_devices_after_reset()
10530 		if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10537 			ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10547 			ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",  in _scsih_scan_for_devices_after_reset()
10549 			_scsih_sas_volume_add(ioc, &element);  in _scsih_scan_for_devices_after_reset()
10550 			ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",  in _scsih_scan_for_devices_after_reset()
10555 	ioc_info(ioc, "\tscan devices: volumes complete\n");  in _scsih_scan_for_devices_after_reset()
10559 	ioc_info(ioc, "\tscan devices: end devices start\n");  in _scsih_scan_for_devices_after_reset()
10563 	while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10569 			ioc_info(ioc, "\tbreak from end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10578 		sas_device = mpt3sas_get_sdev_by_addr(ioc,  in _scsih_scan_for_devices_after_reset()
10580 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10586 		if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address)) {  in _scsih_scan_for_devices_after_reset()
10587 			ioc_info(ioc, "\tBEFORE adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10590 			mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_scan_for_devices_after_reset()
10592 			    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10598 			while (_scsih_add_device(ioc, handle, retry_count++,  in _scsih_scan_for_devices_after_reset()
10602 			ioc_info(ioc, "\tAFTER adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10607 	ioc_info(ioc, "\tscan devices: end devices complete\n");  in _scsih_scan_for_devices_after_reset()
10608 	ioc_info(ioc, "\tscan devices: pcie end devices start\n");  in _scsih_scan_for_devices_after_reset()
10612 	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10618 			ioc_info(ioc, "\tbreak from pcie end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10626 		pcie_device = mpt3sas_get_pdev_by_wwid(ioc,  in _scsih_scan_for_devices_after_reset()
10634 		_scsih_pcie_add_device(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10636 		ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10643 	ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");  in _scsih_scan_for_devices_after_reset()
10644 	ioc_info(ioc, "scan devices: complete\n");  in _scsih_scan_for_devices_after_reset()
10653 void mpt3sas_scsih_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_pre_reset_handler()  argument
10655 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));  in mpt3sas_scsih_pre_reset_handler()
10666 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()  argument
10668 	dtmprintk(ioc,  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10669 	    ioc_info(ioc, "%s: clear outstanding scsi & tm cmds\n", __func__));  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10670 	if (ioc->scsih_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10671 		ioc->scsih_cmds.status |= MPT3_CMD_RESET;  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10672 		mpt3sas_base_free_smid(ioc, ioc->scsih_cmds.smid);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10673 		complete(&ioc->scsih_cmds.done);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10675 	if (ioc->tm_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10676 		ioc->tm_cmds.status |= MPT3_CMD_RESET;  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10677 		mpt3sas_base_free_smid(ioc, ioc->tm_cmds.smid);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10678 		complete(&ioc->tm_cmds.done);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10681 	memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10682 	memset(ioc->device_remove_in_progress, 0,  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10683 	       ioc->device_remove_in_progress_sz);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10684 	_scsih_fw_event_cleanup_queue(ioc);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10685 	_scsih_flush_running_cmds(ioc);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10695 mpt3sas_scsih_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_reset_done_handler()  argument
10697 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));  in mpt3sas_scsih_reset_done_handler()
10698 	if (!(disable_discovery > 0 && !ioc->sas_hba.num_phys)) {  in mpt3sas_scsih_reset_done_handler()
10699 		if (ioc->multipath_on_hba) {  in mpt3sas_scsih_reset_done_handler()
10700 			_scsih_sas_port_refresh(ioc);  in mpt3sas_scsih_reset_done_handler()
10701 			_scsih_update_vphys_after_reset(ioc);  in mpt3sas_scsih_reset_done_handler()
10703 		_scsih_prep_device_scan(ioc);  in mpt3sas_scsih_reset_done_handler()
10704 		_scsih_create_enclosure_list_after_reset(ioc);  in mpt3sas_scsih_reset_done_handler()
10705 		_scsih_search_responding_sas_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10706 		_scsih_search_responding_pcie_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10707 		_scsih_search_responding_raid_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10708 		_scsih_search_responding_expanders(ioc);  in mpt3sas_scsih_reset_done_handler()
10709 		_scsih_error_recovery_delete_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10720 _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)  in _mpt3sas_fw_work()  argument
10722 	ioc->current_event = fw_event;  in _mpt3sas_fw_work()
10723 	_scsih_fw_event_del_from_list(ioc, fw_event);  in _mpt3sas_fw_work()
10726 	if (ioc->remove_host || ioc->pci_error_recovery) {  in _mpt3sas_fw_work()
10728 		ioc->current_event = NULL;  in _mpt3sas_fw_work()
10734 		mpt3sas_process_trigger_data(ioc,  in _mpt3sas_fw_work()
10739 		while (scsi_host_in_recovery(ioc->shost) ||  in _mpt3sas_fw_work()
10740 					 ioc->shost_recovery) {  in _mpt3sas_fw_work()
10745 			if (ioc->remove_host || ioc->fw_events_cleanup)  in _mpt3sas_fw_work()
10749 		_scsih_remove_unresponding_devices(ioc);  in _mpt3sas_fw_work()
10750 		_scsih_del_dirty_vphy(ioc);  in _mpt3sas_fw_work()
10751 		_scsih_del_dirty_port_entries(ioc);  in _mpt3sas_fw_work()
10752 		if (ioc->is_gen35_ioc)  in _mpt3sas_fw_work()
10753 			_scsih_update_device_qdepth(ioc);  in _mpt3sas_fw_work()
10754 		_scsih_scan_for_devices_after_reset(ioc);  in _mpt3sas_fw_work()
10765 		if (ioc->is_driver_loading)  in _mpt3sas_fw_work()
10766 			_scsih_complete_devices_scanning(ioc);  in _mpt3sas_fw_work()
10767 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _mpt3sas_fw_work()
10770 		ioc->start_scan = 0;  in _mpt3sas_fw_work()
10772 			mpt3sas_base_update_missing_delay(ioc, missing_delay[0],  in _mpt3sas_fw_work()
10774 		dewtprintk(ioc,  in _mpt3sas_fw_work()
10775 			   ioc_info(ioc, "port enable: complete from worker thread\n"));  in _mpt3sas_fw_work()
10778 		_scsih_turn_on_pfa_led(ioc, fw_event->device_handle);  in _mpt3sas_fw_work()
10781 		_scsih_sas_topology_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10784 		if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _mpt3sas_fw_work()
10785 			_scsih_sas_device_status_change_event_debug(ioc,  in _mpt3sas_fw_work()
10790 		_scsih_sas_discovery_event(ioc, fw_event);  in _mpt3sas_fw_work()
10793 		_scsih_sas_device_discovery_error_event(ioc, fw_event);  in _mpt3sas_fw_work()
10796 		_scsih_sas_broadcast_primitive_event(ioc, fw_event);  in _mpt3sas_fw_work()
10799 		_scsih_sas_enclosure_dev_status_change_event(ioc,  in _mpt3sas_fw_work()
10803 		_scsih_sas_ir_config_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10806 		_scsih_sas_ir_volume_event(ioc, fw_event);  in _mpt3sas_fw_work()
10809 		_scsih_sas_ir_physical_disk_event(ioc, fw_event);  in _mpt3sas_fw_work()
10812 		_scsih_sas_ir_operation_status_event(ioc, fw_event);  in _mpt3sas_fw_work()
10815 		_scsih_pcie_device_status_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10818 		_scsih_pcie_enumeration_event(ioc, fw_event);  in _mpt3sas_fw_work()
10821 		_scsih_pcie_topology_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10822 		ioc->current_event = NULL;  in _mpt3sas_fw_work()
10827 	ioc->current_event = NULL;  in _mpt3sas_fw_work()
10844 	_mpt3sas_fw_work(fw_event->ioc, fw_event);  in _firmware_event_work()
10861 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,  in mpt3sas_scsih_event_callback()  argument
10871 	if (ioc->pci_error_recovery)  in mpt3sas_scsih_event_callback()
10874 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_scsih_event_callback()
10877 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in mpt3sas_scsih_event_callback()
10885 		mpt3sas_trigger_event(ioc, event, 0);  in mpt3sas_scsih_event_callback()
10899 		if (ioc->broadcast_aen_busy) {  in mpt3sas_scsih_event_callback()
10900 			ioc->broadcast_aen_pending++;  in mpt3sas_scsih_event_callback()
10903 			ioc->broadcast_aen_busy = 1;  in mpt3sas_scsih_event_callback()
10908 		_scsih_check_topo_delete_events(ioc,  in mpt3sas_scsih_event_callback()
10919 		if (ioc->shost_recovery)  in mpt3sas_scsih_event_callback()
10923 	_scsih_check_pcie_topo_remove_events(ioc,  in mpt3sas_scsih_event_callback()
10926 		if (ioc->shost_recovery)  in mpt3sas_scsih_event_callback()
10930 		_scsih_check_ir_config_unhide_events(ioc,  in mpt3sas_scsih_event_callback()
10935 		_scsih_check_volume_delete_events(ioc,  in mpt3sas_scsih_event_callback()
10944 		if (!ioc->is_warpdrive)  in mpt3sas_scsih_event_callback()
10957 …ioc_warn(ioc, "WarpDrive Warning: IO Throttling has occurred in the WarpDrive subsystem. Check War…  in mpt3sas_scsih_event_callback()
10960 …ioc_warn(ioc, "WarpDrive Warning: Program/Erase Cycles for the WarpDrive subsystem in degraded ran…  in mpt3sas_scsih_event_callback()
10963 …ioc_err(ioc, "WarpDrive Fatal Error: There are no Program/Erase Cycles for the WarpDrive subsystem…  in mpt3sas_scsih_event_callback()
10966 …ioc_err(ioc, "WarpDrive Fatal Error: The Backup Rail Monitor has failed on the WarpDrive subsystem…  in mpt3sas_scsih_event_callback()
10973 		_scsih_sas_device_status_change_event(ioc,  in mpt3sas_scsih_event_callback()
10987 		_scsih_temp_threshold_events(ioc,  in mpt3sas_scsih_event_callback()
10996 			ioc_notice(ioc, "Currently an active cable with ReceptacleID %d\n",  in mpt3sas_scsih_event_callback()
11006 			ioc_notice(ioc, "Currently a cable with ReceptacleID %d\n",  in mpt3sas_scsih_event_callback()
11022 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_scsih_event_callback()
11028 	fw_event->ioc = ioc;  in mpt3sas_scsih_event_callback()
11032 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_scsih_event_callback()
11046 _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_expander_node_remove()  argument
11056 		if (ioc->shost_recovery)  in _scsih_expander_node_remove()
11060 			mpt3sas_device_remove_by_sas_address(ioc,  in _scsih_expander_node_remove()
11067 			mpt3sas_expander_remove(ioc,  in _scsih_expander_node_remove()
11074 	mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,  in _scsih_expander_node_remove()
11077 	ioc_info(ioc,  in _scsih_expander_node_remove()
11083 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_node_remove()
11085 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_node_remove()
11100 _scsih_nvme_shutdown(struct MPT3SAS_ADAPTER *ioc)  in _scsih_nvme_shutdown()  argument
11107 	if (list_empty(&ioc->pcie_device_list))  in _scsih_nvme_shutdown()
11110 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_nvme_shutdown()
11112 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_nvme_shutdown()
11113 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_nvme_shutdown()
11117 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_nvme_shutdown()
11119 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_nvme_shutdown()
11121 		ioc_err(ioc,  in _scsih_nvme_shutdown()
11123 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_nvme_shutdown()
11127 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_nvme_shutdown()
11128 	ioc->scsih_cmds.smid = smid;  in _scsih_nvme_shutdown()
11133 	init_completion(&ioc->scsih_cmds.done);  in _scsih_nvme_shutdown()
11134 	ioc->put_smid_default(ioc, smid);  in _scsih_nvme_shutdown()
11136 	ioc_info(ioc,  in _scsih_nvme_shutdown()
11138 		ioc->max_shutdown_latency);  in _scsih_nvme_shutdown()
11139 	wait_for_completion_timeout(&ioc->scsih_cmds.done,  in _scsih_nvme_shutdown()
11140 			ioc->max_shutdown_latency*HZ);  in _scsih_nvme_shutdown()
11142 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_nvme_shutdown()
11143 		ioc_err(ioc, "%s: timeout\n", __func__);  in _scsih_nvme_shutdown()
11147 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_nvme_shutdown()
11148 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_nvme_shutdown()
11149 		ioc_info(ioc, "Io Unit Control shutdown (complete):"  in _scsih_nvme_shutdown()
11155 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_nvme_shutdown()
11156 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_nvme_shutdown()
11168 _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)  in _scsih_ir_shutdown()  argument
11175 	if (!ioc->ir_firmware)  in _scsih_ir_shutdown()
11179 	if (list_empty(&ioc->raid_device_list))  in _scsih_ir_shutdown()
11182 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_ir_shutdown()
11184 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_ir_shutdown()
11185 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_ir_shutdown()
11188 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_ir_shutdown()
11190 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_ir_shutdown()
11192 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _scsih_ir_shutdown()
11193 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_shutdown()
11197 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_ir_shutdown()
11198 	ioc->scsih_cmds.smid = smid;  in _scsih_ir_shutdown()
11204 	if (!ioc->hide_ir_msg)  in _scsih_ir_shutdown()
11205 		ioc_info(ioc, "IR shutdown (sending)\n");  in _scsih_ir_shutdown()
11206 	init_completion(&ioc->scsih_cmds.done);  in _scsih_ir_shutdown()
11207 	ioc->put_smid_default(ioc, smid);  in _scsih_ir_shutdown()
11208 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);  in _scsih_ir_shutdown()
11210 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_ir_shutdown()
11211 		ioc_err(ioc, "%s: timeout\n", __func__);  in _scsih_ir_shutdown()
11215 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_ir_shutdown()
11216 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_ir_shutdown()
11217 		if (!ioc->hide_ir_msg)  in _scsih_ir_shutdown()
11218 			ioc_info(ioc, "IR shutdown (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_ir_shutdown()
11224 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_shutdown()
11225 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_ir_shutdown()
11239 	struct Scsi_Host **shost, struct MPT3SAS_ADAPTER **ioc)  in _scsih_get_shost_and_ioc()  argument
11247 	*ioc = shost_priv(*shost);  in _scsih_get_shost_and_ioc()
11248 	if (*ioc == NULL) {  in _scsih_get_shost_and_ioc()
11265 	struct MPT3SAS_ADAPTER *ioc;  in scsih_remove()  local
11275 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_remove()
11278 	ioc->remove_host = 1;  in scsih_remove()
11281 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_remove()
11282 		_scsih_flush_running_cmds(ioc);  in scsih_remove()
11285 	_scsih_fw_event_cleanup_queue(ioc);  in scsih_remove()
11287 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in scsih_remove()
11288 	wq = ioc->firmware_event_thread;  in scsih_remove()
11289 	ioc->firmware_event_thread = NULL;  in scsih_remove()
11290 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in scsih_remove()
11297 	if (ioc->is_aero_ioc)  in scsih_remove()
11298 		mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,  in scsih_remove()
11299 				&ioc->ioc_pg1_copy);  in scsih_remove()
11301 	_scsih_ir_shutdown(ioc);  in scsih_remove()
11302 	mpt3sas_destroy_debugfs(ioc);  in scsih_remove()
11304 	list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,  in scsih_remove()
11312 		ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in scsih_remove()
11314 		_scsih_raid_device_remove(ioc, raid_device);  in scsih_remove()
11316 	list_for_each_entry_safe(pcie_device, pcienext, &ioc->pcie_device_list,  in scsih_remove()
11318 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in scsih_remove()
11325 	   &ioc->sas_hba.sas_port_list, port_list) {  in scsih_remove()
11328 			mpt3sas_device_remove_by_sas_address(ioc,  in scsih_remove()
11335 			mpt3sas_expander_remove(ioc,  in scsih_remove()
11341 	    &ioc->port_table_list, list) {  in scsih_remove()
11347 	if (ioc->sas_hba.num_phys) {  in scsih_remove()
11348 		kfree(ioc->sas_hba.phy);  in scsih_remove()
11349 		ioc->sas_hba.phy = NULL;  in scsih_remove()
11350 		ioc->sas_hba.num_phys = 0;  in scsih_remove()
11353 	mpt3sas_base_detach(ioc);  in scsih_remove()
11355 	list_del(&ioc->list);  in scsih_remove()
11368 	struct MPT3SAS_ADAPTER *ioc;  in scsih_shutdown()  local
11373 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_shutdown()
11376 	ioc->remove_host = 1;  in scsih_shutdown()
11379 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_shutdown()
11380 		_scsih_flush_running_cmds(ioc);  in scsih_shutdown()
11383 	_scsih_fw_event_cleanup_queue(ioc);  in scsih_shutdown()
11385 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in scsih_shutdown()
11386 	wq = ioc->firmware_event_thread;  in scsih_shutdown()
11387 	ioc->firmware_event_thread = NULL;  in scsih_shutdown()
11388 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in scsih_shutdown()
11395 	if (ioc->is_aero_ioc)  in scsih_shutdown()
11396 		mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,  in scsih_shutdown()
11397 				&ioc->ioc_pg1_copy);  in scsih_shutdown()
11399 	_scsih_ir_shutdown(ioc);  in scsih_shutdown()
11400 	_scsih_nvme_shutdown(ioc);  in scsih_shutdown()
11401 	mpt3sas_base_mask_interrupts(ioc);  in scsih_shutdown()
11402 	mpt3sas_base_stop_watchdog(ioc);  in scsih_shutdown()
11403 	ioc->shost_recovery = 1;  in scsih_shutdown()
11404 	mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in scsih_shutdown()
11405 	ioc->shost_recovery = 0;  in scsih_shutdown()
11406 	mpt3sas_base_free_irq(ioc);  in scsih_shutdown()
11407 	mpt3sas_base_disable_msix(ioc);  in scsih_shutdown()
11420 _scsih_probe_boot_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_boot_devices()  argument
11436 	if (!ioc->bios_pg3.BiosVersion)  in _scsih_probe_boot_devices()
11440 	if (ioc->req_boot_device.device) {  in _scsih_probe_boot_devices()
11441 		device =  ioc->req_boot_device.device;  in _scsih_probe_boot_devices()
11442 		channel = ioc->req_boot_device.channel;  in _scsih_probe_boot_devices()
11443 	} else if (ioc->req_alt_boot_device.device) {  in _scsih_probe_boot_devices()
11444 		device =  ioc->req_alt_boot_device.device;  in _scsih_probe_boot_devices()
11445 		channel = ioc->req_alt_boot_device.channel;  in _scsih_probe_boot_devices()
11446 	} else if (ioc->current_boot_device.device) {  in _scsih_probe_boot_devices()
11447 		device =  ioc->current_boot_device.device;  in _scsih_probe_boot_devices()
11448 		channel = ioc->current_boot_device.channel;  in _scsih_probe_boot_devices()
11463 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_probe_boot_devices()
11466 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_probe_boot_devices()
11476 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_probe_boot_devices()
11478 		list_move_tail(&pcie_device->list, &ioc->pcie_device_list);  in _scsih_probe_boot_devices()
11479 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_probe_boot_devices()
11480 		rc = scsi_add_device(ioc->shost, PCIE_CHANNEL, tid, 0);  in _scsih_probe_boot_devices()
11482 			_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_boot_devices()
11492 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_probe_boot_devices()
11497 		list_move_tail(&sas_device->list, &ioc->sas_device_list);  in _scsih_probe_boot_devices()
11498 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_probe_boot_devices()
11500 		if (ioc->hide_drives)  in _scsih_probe_boot_devices()
11506 		if (!mpt3sas_transport_port_add(ioc, handle,  in _scsih_probe_boot_devices()
11508 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_boot_devices()
11510 			if (!ioc->is_driver_loading) {  in _scsih_probe_boot_devices()
11511 				mpt3sas_transport_port_remove(ioc,  in _scsih_probe_boot_devices()
11514 				_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_boot_devices()
11527 _scsih_probe_raid(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_raid()  argument
11533 	    &ioc->raid_device_list, list) {  in _scsih_probe_raid()
11536 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_probe_raid()
11539 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_probe_raid()
11543 static struct _sas_device *get_next_sas_device(struct MPT3SAS_ADAPTER *ioc)  in get_next_sas_device()  argument
11548 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in get_next_sas_device()
11549 	if (!list_empty(&ioc->sas_device_init_list)) {  in get_next_sas_device()
11550 		sas_device = list_first_entry(&ioc->sas_device_init_list,  in get_next_sas_device()
11554 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in get_next_sas_device()
11559 static void sas_device_make_active(struct MPT3SAS_ADAPTER *ioc,  in sas_device_make_active()  argument
11564 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in sas_device_make_active()
11580 	list_add_tail(&sas_device->list, &ioc->sas_device_list);  in sas_device_make_active()
11582 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in sas_device_make_active()
11592 _scsih_probe_sas(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_sas()  argument
11596 	if (ioc->hide_drives)  in _scsih_probe_sas()
11599 	while ((sas_device = get_next_sas_device(ioc))) {  in _scsih_probe_sas()
11600 		if (!mpt3sas_transport_port_add(ioc, sas_device->handle,  in _scsih_probe_sas()
11602 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_sas()
11612 			if (!ioc->is_driver_loading) {  in _scsih_probe_sas()
11613 				mpt3sas_transport_port_remove(ioc,  in _scsih_probe_sas()
11617 				_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_sas()
11622 		sas_device_make_active(ioc, sas_device);  in _scsih_probe_sas()
11636 static struct _pcie_device *get_next_pcie_device(struct MPT3SAS_ADAPTER *ioc)  in get_next_pcie_device()  argument
11641 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in get_next_pcie_device()
11642 	if (!list_empty(&ioc->pcie_device_init_list)) {  in get_next_pcie_device()
11643 		pcie_device = list_first_entry(&ioc->pcie_device_init_list,  in get_next_pcie_device()
11647 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in get_next_pcie_device()
11660 static void pcie_device_make_active(struct MPT3SAS_ADAPTER *ioc,  in pcie_device_make_active()  argument
11665 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in pcie_device_make_active()
11672 	list_add_tail(&pcie_device->list, &ioc->pcie_device_list);  in pcie_device_make_active()
11674 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in pcie_device_make_active()
11684 _scsih_probe_pcie(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_pcie()  argument
11690 	while ((pcie_device = get_next_pcie_device(ioc))) {  in _scsih_probe_pcie()
11697 			pcie_device_make_active(ioc, pcie_device);  in _scsih_probe_pcie()
11701 		rc = scsi_add_device(ioc->shost, PCIE_CHANNEL,  in _scsih_probe_pcie()
11704 			_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_pcie()
11714 			if (!ioc->is_driver_loading) {  in _scsih_probe_pcie()
11718 				_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_pcie()
11723 		pcie_device_make_active(ioc, pcie_device);  in _scsih_probe_pcie()
11735 _scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_devices()  argument
11739 	if (!(ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR))  in _scsih_probe_devices()
11742 	_scsih_probe_boot_devices(ioc);  in _scsih_probe_devices()
11744 	if (ioc->ir_firmware) {  in _scsih_probe_devices()
11746 		    le16_to_cpu(ioc->ioc_pg8.IRVolumeMappingFlags) &  in _scsih_probe_devices()
11750 			_scsih_probe_raid(ioc);  in _scsih_probe_devices()
11751 			_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11753 			_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11754 			_scsih_probe_raid(ioc);  in _scsih_probe_devices()
11757 		_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11758 		_scsih_probe_pcie(ioc);  in _scsih_probe_devices()
11773 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_scan_start()  local
11776 		mpt3sas_enable_diag_buffer(ioc, diag_buffer_enable);  in scsih_scan_start()
11777 	else if (ioc->manu_pg11.HostTraceBufferMaxSizeKB != 0)  in scsih_scan_start()
11778 		mpt3sas_enable_diag_buffer(ioc, 1);  in scsih_scan_start()
11783 	ioc->start_scan = 1;  in scsih_scan_start()
11784 	rc = mpt3sas_port_enable(ioc);  in scsih_scan_start()
11787 		ioc_info(ioc, "port enable: FAILED\n");  in scsih_scan_start()
11797 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc)  in _scsih_complete_devices_scanning()  argument
11800 	if (ioc->wait_for_discovery_to_complete) {  in _scsih_complete_devices_scanning()
11801 		ioc->wait_for_discovery_to_complete = 0;  in _scsih_complete_devices_scanning()
11802 		_scsih_probe_devices(ioc);  in _scsih_complete_devices_scanning()
11805 	mpt3sas_base_start_watchdog(ioc);  in _scsih_complete_devices_scanning()
11806 	ioc->is_driver_loading = 0;  in _scsih_complete_devices_scanning()
11821 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_scan_finished()  local
11826 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11827 		ioc->wait_for_discovery_to_complete = 0;  in scsih_scan_finished()
11832 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11833 		ioc_info(ioc, "port enable: FAILED with timeout (timeout=300s)\n");  in scsih_scan_finished()
11834 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11838 	if (ioc->start_scan) {  in scsih_scan_finished()
11839 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in scsih_scan_finished()
11841 			mpt3sas_print_fault_code(ioc, ioc_state &  in scsih_scan_finished()
11847 			mpt3sas_base_coredump_info(ioc, ioc_state &  in scsih_scan_finished()
11849 			mpt3sas_base_wait_for_coredump_completion(ioc, __func__);  in scsih_scan_finished()
11856 	if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) {  in scsih_scan_finished()
11857 		ioc_info(ioc,  in scsih_scan_finished()
11859 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11862 	if (ioc->start_scan_failed) {  in scsih_scan_finished()
11863 		ioc_info(ioc, "port enable: FAILED with (ioc_status=0x%08x)\n",  in scsih_scan_finished()
11864 			 ioc->start_scan_failed);  in scsih_scan_finished()
11865 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11866 		ioc->wait_for_discovery_to_complete = 0;  in scsih_scan_finished()
11867 		ioc->remove_host = 1;  in scsih_scan_finished()
11871 	ioc_info(ioc, "port enable: SUCCESS\n");  in scsih_scan_finished()
11872 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11873 	_scsih_complete_devices_scanning(ioc);  in scsih_scan_finished()
11877 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11878 		if (mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET))  in scsih_scan_finished()
11879 			ioc->is_driver_loading = 0;  in scsih_scan_finished()
11890 	struct MPT3SAS_ADAPTER *ioc =  in scsih_map_queues()  local
11894 	int nr_msix_vectors = ioc->iopoll_q_start_index;  in scsih_map_queues()
11895 	int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors;  in scsih_map_queues()
11906 			    nr_msix_vectors - ioc->high_iops_queues;  in scsih_map_queues()
11907 			offset = ioc->high_iops_queues;  in scsih_map_queues()
11920 			blk_mq_pci_map_queues(map, ioc->pdev, offset);  in scsih_map_queues()
12089 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_probe()  local
12122 		ioc = shost_priv(shost);  in _scsih_probe()
12123 		memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));  in _scsih_probe()
12124 		ioc->hba_mpi_version_belonged = hba_mpi_version;  in _scsih_probe()
12125 		ioc->id = mpt2_ids++;  in _scsih_probe()
12126 		sprintf(ioc->driver_name, "%s", MPT2SAS_DRIVER_NAME);  in _scsih_probe()
12129 			ioc->is_warpdrive = 1;  in _scsih_probe()
12130 			ioc->hide_ir_msg = 1;  in _scsih_probe()
12134 			ioc->is_mcpu_endpoint = 1;  in _scsih_probe()
12137 			ioc->mfg_pg10_hide_flag = MFG_PAGE10_EXPOSE_ALL_DISKS;  in _scsih_probe()
12142 			ioc->multipath_on_hba = 0;  in _scsih_probe()
12144 			ioc->multipath_on_hba = 1;  in _scsih_probe()
12154 		ioc = shost_priv(shost);  in _scsih_probe()
12155 		memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));  in _scsih_probe()
12156 		ioc->hba_mpi_version_belonged = hba_mpi_version;  in _scsih_probe()
12157 		ioc->id = mpt3_ids++;  in _scsih_probe()
12158 		sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME);  in _scsih_probe()
12168 			ioc->is_gen35_ioc = 1;  in _scsih_probe()
12191 			ioc->is_aero_ioc = ioc->is_gen35_ioc = 1;  in _scsih_probe()
12194 			ioc->is_gen35_ioc = ioc->is_aero_ioc = 0;  in _scsih_probe()
12196 		if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&  in _scsih_probe()
12198 			(ioc->hba_mpi_version_belonged == MPI26_VERSION)) {  in _scsih_probe()
12199 			ioc->combined_reply_queue = 1;  in _scsih_probe()
12200 			if (ioc->is_gen35_ioc)  in _scsih_probe()
12201 				ioc->combined_reply_index_count =  in _scsih_probe()
12204 				ioc->combined_reply_index_count =  in _scsih_probe()
12208 		switch (ioc->is_gen35_ioc) {  in _scsih_probe()
12211 				ioc->multipath_on_hba = 0;  in _scsih_probe()
12213 				ioc->multipath_on_hba = 1;  in _scsih_probe()
12217 				ioc->multipath_on_hba = 1;  in _scsih_probe()
12219 				ioc->multipath_on_hba = 0;  in _scsih_probe()
12230 	INIT_LIST_HEAD(&ioc->list);  in _scsih_probe()
12232 	list_add_tail(&ioc->list, &mpt3sas_ioc_list);  in _scsih_probe()
12234 	ioc->shost = shost;  in _scsih_probe()
12235 	ioc->pdev = pdev;  in _scsih_probe()
12236 	ioc->scsi_io_cb_idx = scsi_io_cb_idx;  in _scsih_probe()
12237 	ioc->tm_cb_idx = tm_cb_idx;  in _scsih_probe()
12238 	ioc->ctl_cb_idx = ctl_cb_idx;  in _scsih_probe()
12239 	ioc->base_cb_idx = base_cb_idx;  in _scsih_probe()
12240 	ioc->port_enable_cb_idx = port_enable_cb_idx;  in _scsih_probe()
12241 	ioc->transport_cb_idx = transport_cb_idx;  in _scsih_probe()
12242 	ioc->scsih_cb_idx = scsih_cb_idx;  in _scsih_probe()
12243 	ioc->config_cb_idx = config_cb_idx;  in _scsih_probe()
12244 	ioc->tm_tr_cb_idx = tm_tr_cb_idx;  in _scsih_probe()
12245 	ioc->tm_tr_volume_cb_idx = tm_tr_volume_cb_idx;  in _scsih_probe()
12246 	ioc->tm_sas_control_cb_idx = tm_sas_control_cb_idx;  in _scsih_probe()
12247 	ioc->logging_level = logging_level;  in _scsih_probe()
12248 	ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;  in _scsih_probe()
12250 	ioc->max_shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;  in _scsih_probe()
12254 	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE;  in _scsih_probe()
12256 	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY;  in _scsih_probe()
12258 	ioc->enable_sdev_max_qd = enable_sdev_max_qd;  in _scsih_probe()
12261 	mutex_init(&ioc->reset_in_progress_mutex);  in _scsih_probe()
12263 	mutex_init(&ioc->pci_access_mutex);  in _scsih_probe()
12264 	spin_lock_init(&ioc->ioc_reset_in_progress_lock);  in _scsih_probe()
12265 	spin_lock_init(&ioc->scsi_lookup_lock);  in _scsih_probe()
12266 	spin_lock_init(&ioc->sas_device_lock);  in _scsih_probe()
12267 	spin_lock_init(&ioc->sas_node_lock);  in _scsih_probe()
12268 	spin_lock_init(&ioc->fw_event_lock);  in _scsih_probe()
12269 	spin_lock_init(&ioc->raid_device_lock);  in _scsih_probe()
12270 	spin_lock_init(&ioc->pcie_device_lock);  in _scsih_probe()
12271 	spin_lock_init(&ioc->diag_trigger_lock);  in _scsih_probe()
12273 	INIT_LIST_HEAD(&ioc->sas_device_list);  in _scsih_probe()
12274 	INIT_LIST_HEAD(&ioc->sas_device_init_list);  in _scsih_probe()
12275 	INIT_LIST_HEAD(&ioc->sas_expander_list);  in _scsih_probe()
12276 	INIT_LIST_HEAD(&ioc->enclosure_list);  in _scsih_probe()
12277 	INIT_LIST_HEAD(&ioc->pcie_device_list);  in _scsih_probe()
12278 	INIT_LIST_HEAD(&ioc->pcie_device_init_list);  in _scsih_probe()
12279 	INIT_LIST_HEAD(&ioc->fw_event_list);  in _scsih_probe()
12280 	INIT_LIST_HEAD(&ioc->raid_device_list);  in _scsih_probe()
12281 	INIT_LIST_HEAD(&ioc->sas_hba.sas_port_list);  in _scsih_probe()
12282 	INIT_LIST_HEAD(&ioc->delayed_tr_list);  in _scsih_probe()
12283 	INIT_LIST_HEAD(&ioc->delayed_sc_list);  in _scsih_probe()
12284 	INIT_LIST_HEAD(&ioc->delayed_event_ack_list);  in _scsih_probe()
12285 	INIT_LIST_HEAD(&ioc->delayed_tr_volume_list);  in _scsih_probe()
12286 	INIT_LIST_HEAD(&ioc->reply_queue_list);  in _scsih_probe()
12287 	INIT_LIST_HEAD(&ioc->port_table_list);  in _scsih_probe()
12289 	sprintf(ioc->name, "%s_cm%d", ioc->driver_name, ioc->id);  in _scsih_probe()
12295 	shost->unique_id = ioc->id;  in _scsih_probe()
12297 	if (ioc->is_mcpu_endpoint) {  in _scsih_probe()
12300 		ioc_info(ioc, "The max_sectors value is set to %d\n",  in _scsih_probe()
12306 …ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767. Assigning value of 6…  in _scsih_probe()
12310 …ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767.Assigning default val…  in _scsih_probe()
12314 				ioc_info(ioc, "The max_sectors value is set to %d\n",  in _scsih_probe()
12330 	snprintf(ioc->firmware_event_name, sizeof(ioc->firmware_event_name),  in _scsih_probe()
12331 	    "fw_event_%s%d", ioc->driver_name, ioc->id);  in _scsih_probe()
12332 	ioc->firmware_event_thread = alloc_ordered_workqueue(  in _scsih_probe()
12333 	    ioc->firmware_event_name, 0);  in _scsih_probe()
12334 	if (!ioc->firmware_event_thread) {  in _scsih_probe()
12335 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12343 	if (ioc->is_gen35_ioc && host_tagset_enable)  in _scsih_probe()
12346 	ioc->is_driver_loading = 1;  in _scsih_probe()
12347 	if ((mpt3sas_base_attach(ioc))) {  in _scsih_probe()
12348 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12354 	if (ioc->is_warpdrive) {  in _scsih_probe()
12355 		if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_EXPOSE_ALL_DISKS)  in _scsih_probe()
12356 			ioc->hide_drives = 0;  in _scsih_probe()
12357 		else if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_HIDE_ALL_DISKS)  in _scsih_probe()
12358 			ioc->hide_drives = 1;  in _scsih_probe()
12360 			if (mpt3sas_get_num_volumes(ioc))  in _scsih_probe()
12361 				ioc->hide_drives = 1;  in _scsih_probe()
12363 				ioc->hide_drives = 0;  in _scsih_probe()
12366 		ioc->hide_drives = 0;  in _scsih_probe()
12372 		    ioc->reply_queue_count - ioc->high_iops_queues;  in _scsih_probe()
12375 		    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in _scsih_probe()
12379 		dev_info(&ioc->pdev->dev,  in _scsih_probe()
12386 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12392 	mpt3sas_setup_debugfs(ioc);  in _scsih_probe()
12395 	mpt3sas_base_detach(ioc);  in _scsih_probe()
12397 	destroy_workqueue(ioc->firmware_event_thread);  in _scsih_probe()
12400 	list_del(&ioc->list);  in _scsih_probe()
12417 	struct MPT3SAS_ADAPTER *ioc;  in scsih_suspend()  local
12420 	rc = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);  in scsih_suspend()
12424 	mpt3sas_base_stop_watchdog(ioc);  in scsih_suspend()
12426 	_scsih_nvme_shutdown(ioc);  in scsih_suspend()
12427 	ioc_info(ioc, "pdev=0x%p, slot=%s, entering operating state\n",  in scsih_suspend()
12430 	mpt3sas_base_free_resources(ioc);  in scsih_suspend()
12445 	struct MPT3SAS_ADAPTER *ioc;  in scsih_resume()  local
12449 	r = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);  in scsih_resume()
12453 	ioc_info(ioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",  in scsih_resume()
12456 	ioc->pdev = pdev;  in scsih_resume()
12457 	r = mpt3sas_base_map_resources(ioc);  in scsih_resume()
12460 	ioc_info(ioc, "Issuing Hard Reset as part of OS Resume\n");  in scsih_resume()
12461 	mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET);  in scsih_resume()
12463 	mpt3sas_base_start_watchdog(ioc);  in scsih_resume()
12480 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_error_detected()  local
12482 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_error_detected()
12485 	ioc_info(ioc, "PCI error: detected callback, state(%d)!!\n", state);  in scsih_pci_error_detected()
12492 		ioc->pci_error_recovery = 1;  in scsih_pci_error_detected()
12493 		scsi_block_requests(ioc->shost);  in scsih_pci_error_detected()
12494 		mpt3sas_base_stop_watchdog(ioc);  in scsih_pci_error_detected()
12495 		mpt3sas_base_free_resources(ioc);  in scsih_pci_error_detected()
12499 		ioc->pci_error_recovery = 1;  in scsih_pci_error_detected()
12500 		mpt3sas_base_stop_watchdog(ioc);  in scsih_pci_error_detected()
12501 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_pci_error_detected()
12502 		_scsih_flush_running_cmds(ioc);  in scsih_pci_error_detected()
12520 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_slot_reset()  local
12523 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_slot_reset()
12526 	ioc_info(ioc, "PCI error: slot reset callback!!\n");  in scsih_pci_slot_reset()
12528 	ioc->pci_error_recovery = 0;  in scsih_pci_slot_reset()
12529 	ioc->pdev = pdev;  in scsih_pci_slot_reset()
12531 	rc = mpt3sas_base_map_resources(ioc);  in scsih_pci_slot_reset()
12535 	ioc_info(ioc, "Issuing Hard Reset as part of PCI Slot Reset\n");  in scsih_pci_slot_reset()
12536 	rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in scsih_pci_slot_reset()
12538 	ioc_warn(ioc, "hard reset: %s\n",  in scsih_pci_slot_reset()
12559 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_resume()  local
12561 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_resume()
12564 	ioc_info(ioc, "PCI error: resume callback!!\n");  in scsih_pci_resume()
12566 	mpt3sas_base_start_watchdog(ioc);  in scsih_pci_resume()
12567 	scsi_unblock_requests(ioc->shost);  in scsih_pci_resume()
12578 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_mmio_enabled()  local
12580 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_mmio_enabled()
12583 	ioc_info(ioc, "PCI error: mmio enabled callback!!\n");  in scsih_pci_mmio_enabled()