135 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc,
138 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
140 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc);
155 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_check_cmd_timeout()  argument
163 	ioc_err(ioc, "Command %s\n",  in mpt3sas_base_check_cmd_timeout()
181 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_set_fwfault_debug()  local
189 	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)  in _scsih_set_fwfault_debug()
190 		ioc->fwfault_debug = mpt3sas_fwfault_debug;  in _scsih_set_fwfault_debug()
232 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,  in _base_clone_reply_to_sys_mem()  argument
240 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_clone_reply_to_sys_mem()
241 	void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +  in _base_clone_reply_to_sys_mem()
243 			(cmd_credit * ioc->request_sz) + (index * sizeof(u32));  in _base_clone_reply_to_sys_mem()
296 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_get_chain()  argument
300 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_chain()
302 	base_chain  = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +  in _base_get_chain()
303 		(cmd_credit * ioc->request_sz) +  in _base_get_chain()
305 	chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *  in _base_get_chain()
306 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);  in _base_get_chain()
322 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_get_chain_phys()  argument
326 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_chain_phys()
328 	base_chain_phys  = ioc->chip_phys + MPI_FRAME_START_OFFSET +  in _base_get_chain_phys()
329 		(cmd_credit * ioc->request_sz) +  in _base_get_chain_phys()
331 	chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *  in _base_get_chain_phys()
332 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);  in _base_get_chain_phys()
348 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_buffer_bar0()  argument
350 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_buffer_bar0()
352 	void __iomem *chain_end = _base_get_chain(ioc,  in _base_get_buffer_bar0()
354 			ioc->facts.MaxChainDepth);  in _base_get_buffer_bar0()
369 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_buffer_phys_bar0()  argument
371 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_buffer_phys_bar0()
372 	phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,  in _base_get_buffer_phys_bar0()
374 			ioc->facts.MaxChainDepth);  in _base_get_buffer_phys_bar0()
390 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,  in _base_get_chain_buffer_dma_to_chain_buffer()  argument
396 	for (index = 0; index < ioc->scsiio_depth; index++) {  in _base_get_chain_buffer_dma_to_chain_buffer()
397 		for (j = 0; j < ioc->chains_needed_per_io; j++) {  in _base_get_chain_buffer_dma_to_chain_buffer()
398 			ct = &ioc->chain_lookup[index].chains_per_smid[j];  in _base_get_chain_buffer_dma_to_chain_buffer()
403 	ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n");  in _base_get_chain_buffer_dma_to_chain_buffer()
417 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,  in _clone_sg_entries()  argument
457 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _clone_sg_entries()
459 			ioc_err(ioc, "scmd is NULL\n");  in _clone_sg_entries()
482 	buffer_iomem = _base_get_buffer_bar0(ioc, smid);  in _clone_sg_entries()
483 	buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);  in _clone_sg_entries()
493 	for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {  in _clone_sg_entries()
506 				_base_get_chain_buffer_dma_to_chain_buffer(ioc,  in _clone_sg_entries()
515 				_base_get_chain(ioc,  in _clone_sg_entries()
519 			dst_addr_phys = _base_get_chain_phys(ioc,  in _clone_sg_entries()
542 					    ioc->config_vaddr,  in _clone_sg_entries()
580 				src_chain_addr[i], ioc->request_sz);  in _clone_sg_entries()
594 	struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;  in mpt3sas_remove_dead_ioc_func()  local
597 	if (!ioc)  in mpt3sas_remove_dead_ioc_func()
600 	pdev = ioc->pdev;  in mpt3sas_remove_dead_ioc_func()
613 static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc)  in _base_sync_drv_fw_timestamp()  argument
622 	mutex_lock(&ioc->scsih_cmds.mutex);  in _base_sync_drv_fw_timestamp()
623 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _base_sync_drv_fw_timestamp()
624 		ioc_err(ioc, "scsih_cmd in use %s\n", __func__);  in _base_sync_drv_fw_timestamp()
627 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _base_sync_drv_fw_timestamp()
628 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _base_sync_drv_fw_timestamp()
630 		ioc_err(ioc, "Failed obtaining a smid %s\n", __func__);  in _base_sync_drv_fw_timestamp()
631 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _base_sync_drv_fw_timestamp()
634 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_sync_drv_fw_timestamp()
635 	ioc->scsih_cmds.smid = smid;  in _base_sync_drv_fw_timestamp()
644 	init_completion(&ioc->scsih_cmds.done);  in _base_sync_drv_fw_timestamp()
645 	ioc->put_smid_default(ioc, smid);  in _base_sync_drv_fw_timestamp()
646 	dinitprintk(ioc, ioc_info(ioc,  in _base_sync_drv_fw_timestamp()
649 	wait_for_completion_timeout(&ioc->scsih_cmds.done,  in _base_sync_drv_fw_timestamp()
651 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_sync_drv_fw_timestamp()
652 		mpt3sas_check_cmd_timeout(ioc,  in _base_sync_drv_fw_timestamp()
653 		    ioc->scsih_cmds.status, mpi_request,  in _base_sync_drv_fw_timestamp()
657 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _base_sync_drv_fw_timestamp()
658 		mpi_reply = ioc->scsih_cmds.reply;  in _base_sync_drv_fw_timestamp()
659 		dinitprintk(ioc, ioc_info(ioc,  in _base_sync_drv_fw_timestamp()
666 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _base_sync_drv_fw_timestamp()
667 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _base_sync_drv_fw_timestamp()
669 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _base_sync_drv_fw_timestamp()
681 	struct MPT3SAS_ADAPTER *ioc =  in _base_fault_reset_work()  local
689 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
690 	if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) ||  in _base_fault_reset_work()
691 			ioc->pci_error_recovery)  in _base_fault_reset_work()
693 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
695 	doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _base_fault_reset_work()
697 		ioc_err(ioc, "SAS host is non-operational !!!!\n");  in _base_fault_reset_work()
707 		if (ioc->non_operational_loop++ < 5) {  in _base_fault_reset_work()
708 			spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,  in _base_fault_reset_work()
720 		mpt3sas_base_pause_mq_polling(ioc);  in _base_fault_reset_work()
721 		ioc->schedule_dead_ioc_flush_running_cmds(ioc);  in _base_fault_reset_work()
726 		ioc->remove_host = 1;  in _base_fault_reset_work()
728 		p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,  in _base_fault_reset_work()
729 		    "%s_dead_ioc_%d", ioc->driver_name, ioc->id);  in _base_fault_reset_work()
731 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",  in _base_fault_reset_work()
734 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n",  in _base_fault_reset_work()
740 		u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?  in _base_fault_reset_work()
741 		    ioc->manu_pg11.CoreDumpTOSec :  in _base_fault_reset_work()
746 		if (ioc->ioc_coredump_loop == 0) {  in _base_fault_reset_work()
747 			mpt3sas_print_coredump_info(ioc,  in _base_fault_reset_work()
751 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
752 			ioc->shost_recovery = 1;  in _base_fault_reset_work()
754 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
755 			mpt3sas_base_mask_interrupts(ioc);  in _base_fault_reset_work()
756 			mpt3sas_base_pause_mq_polling(ioc);  in _base_fault_reset_work()
757 			_base_clear_outstanding_commands(ioc);  in _base_fault_reset_work()
760 		ioc_info(ioc, "%s: CoreDump loop %d.",  in _base_fault_reset_work()
761 		    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
764 		if (ioc->ioc_coredump_loop++ < timeout) {  in _base_fault_reset_work()
766 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
771 	if (ioc->ioc_coredump_loop) {  in _base_fault_reset_work()
773 			ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d",  in _base_fault_reset_work()
774 			    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
776 			ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d",  in _base_fault_reset_work()
777 			    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
778 		ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE;  in _base_fault_reset_work()
780 	ioc->non_operational_loop = 0;  in _base_fault_reset_work()
782 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _base_fault_reset_work()
783 		ioc_warn(ioc, "%s: hard reset: %s\n",  in _base_fault_reset_work()
785 		doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _base_fault_reset_work()
787 			mpt3sas_print_fault_code(ioc, doorbell &  in _base_fault_reset_work()
791 			mpt3sas_print_coredump_info(ioc, doorbell &  in _base_fault_reset_work()
797 	ioc->ioc_coredump_loop = 0;  in _base_fault_reset_work()
798 	if (ioc->time_sync_interval &&  in _base_fault_reset_work()
799 	    ++ioc->timestamp_update_count >= ioc->time_sync_interval) {  in _base_fault_reset_work()
800 		ioc->timestamp_update_count = 0;  in _base_fault_reset_work()
801 		_base_sync_drv_fw_timestamp(ioc);  in _base_fault_reset_work()
803 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
805 	if (ioc->fault_reset_work_q)  in _base_fault_reset_work()
806 		queue_delayed_work(ioc->fault_reset_work_q,  in _base_fault_reset_work()
807 		    &ioc->fault_reset_work,  in _base_fault_reset_work()
809 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
819 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_start_watchdog()  argument
823 	if (ioc->fault_reset_work_q)  in mpt3sas_base_start_watchdog()
826 	ioc->timestamp_update_count = 0;  in mpt3sas_base_start_watchdog()
829 	INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);  in mpt3sas_base_start_watchdog()
830 	snprintf(ioc->fault_reset_work_q_name,  in mpt3sas_base_start_watchdog()
831 	    sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",  in mpt3sas_base_start_watchdog()
832 	    ioc->driver_name, ioc->id);  in mpt3sas_base_start_watchdog()
833 	ioc->fault_reset_work_q =  in mpt3sas_base_start_watchdog()
834 		create_singlethread_workqueue(ioc->fault_reset_work_q_name);  in mpt3sas_base_start_watchdog()
835 	if (!ioc->fault_reset_work_q) {  in mpt3sas_base_start_watchdog()
836 		ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__);  in mpt3sas_base_start_watchdog()
839 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_start_watchdog()
840 	if (ioc->fault_reset_work_q)  in mpt3sas_base_start_watchdog()
841 		queue_delayed_work(ioc->fault_reset_work_q,  in mpt3sas_base_start_watchdog()
842 		    &ioc->fault_reset_work,  in mpt3sas_base_start_watchdog()
844 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_start_watchdog()
854 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_stop_watchdog()  argument
859 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_stop_watchdog()
860 	wq = ioc->fault_reset_work_q;  in mpt3sas_base_stop_watchdog()
861 	ioc->fault_reset_work_q = NULL;  in mpt3sas_base_stop_watchdog()
862 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_stop_watchdog()
864 		if (!cancel_delayed_work_sync(&ioc->fault_reset_work))  in mpt3sas_base_stop_watchdog()
876 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)  in mpt3sas_base_fault_info()  argument
878 	ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code);  in mpt3sas_base_fault_info()
889 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)  in mpt3sas_base_coredump_info()  argument
891 	ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code);  in mpt3sas_base_coredump_info()
903 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_wait_for_coredump_completion()  argument
906 	u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?  in mpt3sas_base_wait_for_coredump_completion()
907 			ioc->manu_pg11.CoreDumpTOSec :  in mpt3sas_base_wait_for_coredump_completion()
910 	int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT,  in mpt3sas_base_wait_for_coredump_completion()
914 		ioc_err(ioc,  in mpt3sas_base_wait_for_coredump_completion()
918 		ioc_info(ioc,  in mpt3sas_base_wait_for_coredump_completion()
935 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_halt_firmware()  argument
939 	if (!ioc->fwfault_debug)  in mpt3sas_halt_firmware()
944 	doorbell = ioc->base_readl(&ioc->chip->Doorbell);  in mpt3sas_halt_firmware()
946 		mpt3sas_print_fault_code(ioc, doorbell &  in mpt3sas_halt_firmware()
950 		mpt3sas_print_coredump_info(ioc, doorbell &  in mpt3sas_halt_firmware()
953 		writel(0xC0FFEE00, &ioc->chip->Doorbell);  in mpt3sas_halt_firmware()
954 		ioc_err(ioc, "Firmware is halted due to command timeout\n");  in mpt3sas_halt_firmware()
957 	if (ioc->fwfault_debug == 2)  in mpt3sas_halt_firmware()
971 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,  in _base_sas_ioc_info()  argument
998 		    !(ioc->logging_level & MPT_DEBUG_CONFIG)) {  in _base_sas_ioc_info()
1157 		frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;  in _base_sas_ioc_info()
1181 		frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;  in _base_sas_ioc_info()
1186 		    ioc->sge_size;  in _base_sas_ioc_info()
1195 	ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",  in _base_sas_ioc_info()
1207 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,  in _base_display_event_data()  argument
1213 	if (!(ioc->logging_level & MPT_DEBUG_EVENTS))  in _base_display_event_data()
1235 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1242 		ioc_info(ioc, "Discovery: (%s)",  in _base_display_event_data()
1267 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1271 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1275 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1279 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1298 		ioc_info(ioc, "PCIE Enumeration: (%s)",  in _base_display_event_data()
1315 	ioc_info(ioc, "%s\n", desc);  in _base_display_event_data()
1324 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc, u32 log_info)  in _base_sas_log_info()  argument
1347 	if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==  in _base_sas_log_info()
1359 		if (!ioc->hide_ir_msg)  in _base_sas_log_info()
1366 	ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",  in _base_sas_log_info()
1379 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in _base_display_reply_info()  argument
1386 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _base_display_reply_info()
1388 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _base_display_reply_info()
1395 	    (ioc->logging_level & MPT_DEBUG_REPLY)) {  in _base_display_reply_info()
1396 		_base_sas_ioc_info(ioc, mpi_reply,  in _base_display_reply_info()
1397 		   mpt3sas_base_get_msg_frame(ioc, smid));  in _base_display_reply_info()
1402 		_base_sas_log_info(ioc, loginfo);  in _base_display_reply_info()
1407 		mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);  in _base_display_reply_info()
1423 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in mpt3sas_base_done()  argument
1428 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_base_done()
1430 		return mpt3sas_check_for_pending_internal_cmds(ioc, smid);  in mpt3sas_base_done()
1432 	if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)  in mpt3sas_base_done()
1435 	ioc->base_cmds.status |= MPT3_CMD_COMPLETE;  in mpt3sas_base_done()
1437 		ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;  in mpt3sas_base_done()
1438 		memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in mpt3sas_base_done()
1440 	ioc->base_cmds.status &= ~MPT3_CMD_PENDING;  in mpt3sas_base_done()
1442 	complete(&ioc->base_cmds.done);  in mpt3sas_base_done()
1457 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)  in _base_async_event()  argument
1464 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _base_async_event()
1470 	_base_display_event_data(ioc, mpi_reply);  in _base_async_event()
1474 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_async_event()
1484 				&ioc->delayed_event_ack_list);  in _base_async_event()
1485 		dewtprintk(ioc,  in _base_async_event()
1486 			   ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n",  in _base_async_event()
1491 	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_async_event()
1498 	ioc->put_smid_default(ioc, smid);  in _base_async_event()
1503 	mpt3sas_scsih_event_callback(ioc, msix_index, reply);  in _base_async_event()
1506 	mpt3sas_ctl_event_callback(ioc, msix_index, reply);  in _base_async_event()
1512 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _get_st_from_smid()  argument
1517 	    WARN_ON(smid >= ioc->hi_priority_smid))  in _get_st_from_smid()
1520 	cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _get_st_from_smid()
1535 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_cb_idx()  argument
1538 	u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;  in _base_get_cb_idx()
1541 	if (smid < ioc->hi_priority_smid) {  in _base_get_cb_idx()
1545 			st = _get_st_from_smid(ioc, smid);  in _base_get_cb_idx()
1549 			cb_idx = ioc->ctl_cb_idx;  in _base_get_cb_idx()
1550 	} else if (smid < ioc->internal_smid) {  in _base_get_cb_idx()
1551 		i = smid - ioc->hi_priority_smid;  in _base_get_cb_idx()
1552 		cb_idx = ioc->hpr_lookup[i].cb_idx;  in _base_get_cb_idx()
1553 	} else if (smid <= ioc->hba_queue_depth) {  in _base_get_cb_idx()
1554 		i = smid - ioc->internal_smid;  in _base_get_cb_idx()
1555 		cb_idx = ioc->internal_lookup[i].cb_idx;  in _base_get_cb_idx()
1572 mpt3sas_base_pause_mq_polling(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_pause_mq_polling()  argument
1575 	    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_pause_mq_polling()
1579 		atomic_set(&ioc->io_uring_poll_queues[qid].pause, 1);  in mpt3sas_base_pause_mq_polling()
1585 		while (atomic_read(&ioc->io_uring_poll_queues[qid].busy)) {  in mpt3sas_base_pause_mq_polling()
1599 mpt3sas_base_resume_mq_polling(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_resume_mq_polling()  argument
1602 	    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_resume_mq_polling()
1606 		atomic_set(&ioc->io_uring_poll_queues[qid].pause, 0);  in mpt3sas_base_resume_mq_polling()
1616 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_mask_interrupts()  argument
1620 	ioc->mask_interrupts = 1;  in mpt3sas_base_mask_interrupts()
1621 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1623 	writel(him_register, &ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1624 	ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1634 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_unmask_interrupts()  argument
1638 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_unmask_interrupts()
1640 	writel(him_register, &ioc->chip->HostInterruptMask);  in mpt3sas_base_unmask_interrupts()
1641 	ioc->mask_interrupts = 0;  in mpt3sas_base_unmask_interrupts()
1680 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;  in _base_process_reply_queue()  local
1709 			cb_idx = _base_get_cb_idx(ioc, smid);  in _base_process_reply_queue()
1712 				rc = mpt_callbacks[cb_idx](ioc, smid,  in _base_process_reply_queue()
1715 					mpt3sas_base_free_smid(ioc, smid);  in _base_process_reply_queue()
1721 			if (reply > ioc->reply_dma_max_address ||  in _base_process_reply_queue()
1722 			    reply < ioc->reply_dma_min_address)  in _base_process_reply_queue()
1725 				cb_idx = _base_get_cb_idx(ioc, smid);  in _base_process_reply_queue()
1728 					rc = mpt_callbacks[cb_idx](ioc, smid,  in _base_process_reply_queue()
1731 						_base_display_reply_info(ioc,  in _base_process_reply_queue()
1734 						mpt3sas_base_free_smid(ioc,  in _base_process_reply_queue()
1738 				_base_async_event(ioc, msix_index, reply);  in _base_process_reply_queue()
1743 				ioc->reply_free_host_index =  in _base_process_reply_queue()
1744 				    (ioc->reply_free_host_index ==  in _base_process_reply_queue()
1745 				    (ioc->reply_free_queue_depth - 1)) ?  in _base_process_reply_queue()
1746 				    0 : ioc->reply_free_host_index + 1;  in _base_process_reply_queue()
1747 				ioc->reply_free[ioc->reply_free_host_index] =  in _base_process_reply_queue()
1749 				if (ioc->is_mcpu_endpoint)  in _base_process_reply_queue()
1750 					_base_clone_reply_to_sys_mem(ioc,  in _base_process_reply_queue()
1752 						ioc->reply_free_host_index);  in _base_process_reply_queue()
1753 				writel(ioc->reply_free_host_index,  in _base_process_reply_queue()
1754 				    &ioc->chip->ReplyFreeHostIndex);  in _base_process_reply_queue()
1761 		    (ioc->reply_post_queue_depth - 1)) ? 0 :  in _base_process_reply_queue()
1772 		if (completed_cmds >= ioc->thresh_hold) {  in _base_process_reply_queue()
1773 			if (ioc->combined_reply_queue) {  in _base_process_reply_queue()
1777 				    ioc->replyPostRegisterIndex[msix_index/8]);  in _base_process_reply_queue()
1782 						&ioc->chip->ReplyPostHostIndex);  in _base_process_reply_queue()
1807 	if (ioc->is_warpdrive) {  in _base_process_reply_queue()
1809 		ioc->reply_post_host_index[msix_index]);  in _base_process_reply_queue()
1829 	if (ioc->combined_reply_queue)  in _base_process_reply_queue()
1832 			ioc->replyPostRegisterIndex[msix_index/8]);  in _base_process_reply_queue()
1836 			&ioc->chip->ReplyPostHostIndex);  in _base_process_reply_queue()
1850 	struct MPT3SAS_ADAPTER *ioc =  in mpt3sas_blk_mq_poll()  local
1854 	int qid = queue_num - ioc->iopoll_q_start_index;  in mpt3sas_blk_mq_poll()
1856 	if (atomic_read(&ioc->io_uring_poll_queues[qid].pause) ||  in mpt3sas_blk_mq_poll()
1857 	    !atomic_add_unless(&ioc->io_uring_poll_queues[qid].busy, 1, 1))  in mpt3sas_blk_mq_poll()
1860 	reply_q = ioc->io_uring_poll_queues[qid].reply_q;  in mpt3sas_blk_mq_poll()
1863 	atomic_dec(&ioc->io_uring_poll_queues[qid].busy);  in mpt3sas_blk_mq_poll()
1879 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;  in _base_interrupt()  local
1881 	if (ioc->mask_interrupts)  in _base_interrupt()
1933 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc)  in _base_init_irqpolls()  argument
1937 	if (list_empty(&ioc->reply_queue_list))  in _base_init_irqpolls()
1940 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {  in _base_init_irqpolls()
1944 			ioc->hba_queue_depth/4, _base_irqpoll);  in _base_init_irqpolls()
1947 		reply_q->os_irq = pci_irq_vector(ioc->pdev,  in _base_init_irqpolls()
1959 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)  in _base_is_controller_msix_enabled()  argument
1961 	return (ioc->facts.IOCCapabilities &  in _base_is_controller_msix_enabled()
1962 	    MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;  in _base_is_controller_msix_enabled()
1975 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll)  in mpt3sas_base_sync_reply_irqs()  argument
1982 	if (!_base_is_controller_msix_enabled(ioc))  in mpt3sas_base_sync_reply_irqs()
1985 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in mpt3sas_base_sync_reply_irqs()
1986 		if (ioc->shost_recovery || ioc->remove_host ||  in mpt3sas_base_sync_reply_irqs()
1987 				ioc->pci_error_recovery)  in mpt3sas_base_sync_reply_irqs()
1998 		synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));  in mpt3sas_base_sync_reply_irqs()
2072 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)  in _base_build_zero_len_sge()  argument
2078 	ioc->base_add_sg_single(paddr, flags_length, -1);  in _base_build_zero_len_sge()
2125 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,  in _base_get_chain_buffer_tracker()  argument
2132 	   atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);  in _base_get_chain_buffer_tracker()
2134 	if (chain_offset == ioc->chains_needed_per_io)  in _base_get_chain_buffer_tracker()
2137 	chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];  in _base_get_chain_buffer_tracker()
2138 	atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);  in _base_get_chain_buffer_tracker()
2153 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,  in _base_build_sg()  argument
2160 		_base_build_zero_len_sge(ioc, psge);  in _base_build_sg()
2169 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2173 		psge += ioc->sge_size;  in _base_build_sg()
2180 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2187 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2194 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2257 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_build_nvme_prp()  argument
2285 	prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);  in _base_build_nvme_prp()
2286 	prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);  in _base_build_nvme_prp()
2292 	page_mask = ioc->page_size - 1;  in _base_build_nvme_prp()
2342 		entry_len = ioc->page_size - offset;  in _base_build_nvme_prp()
2362 			if (length > ioc->page_size) {  in _base_build_nvme_prp()
2424 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,  in base_make_prp_nvme()  argument
2439 	nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);  in base_make_prp_nvme()
2478 	curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);  in base_make_prp_nvme()
2479 	msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);  in base_make_prp_nvme()
2551 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,  in base_is_prp_possible()  argument
2589 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,  in _base_check_pcie_native_sgl()  argument
2601 	if (!base_is_prp_possible(ioc, pcie_device,  in _base_check_pcie_native_sgl()
2610 	base_make_prp_nvme(ioc, scmd, mpi_request,  in _base_check_pcie_native_sgl()
2649 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)  in _base_build_zero_len_sge_ieee()  argument
2673 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,  in _base_build_sg_scmd()  argument
2690 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_build_sg_scmd()
2709 	sges_in_segment = ioc->max_sges_in_main_message;  in _base_build_sg_scmd()
2714 	    (sges_in_segment * ioc->sge_size))/4;  in _base_build_sg_scmd()
2719 			ioc->base_add_sg_single(sg_local,  in _base_build_sg_scmd()
2723 			ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2726 		sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2733 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd()
2740 		    ioc->max_sges_in_chain_message) ? sges_left :  in _base_build_sg_scmd()
2741 		    ioc->max_sges_in_chain_message;  in _base_build_sg_scmd()
2743 		    0 : (sges_in_segment * ioc->sge_size)/4;  in _base_build_sg_scmd()
2744 		chain_length = sges_in_segment * ioc->sge_size;  in _base_build_sg_scmd()
2748 			chain_length += ioc->sge_size;  in _base_build_sg_scmd()
2750 		ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |  in _base_build_sg_scmd()
2759 				ioc->base_add_sg_single(sg_local,  in _base_build_sg_scmd()
2764 				ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2768 			sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2773 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd()
2786 			ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |  in _base_build_sg_scmd()
2789 			ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2792 		sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2814 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,  in _base_build_sg_scmd_ieee()  argument
2830 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_build_sg_scmd_ieee()
2841 	if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,  in _base_build_sg_scmd_ieee()
2853 	sges_in_segment = (ioc->request_sz -  in _base_build_sg_scmd_ieee()
2854 		   offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2859 	    (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);  in _base_build_sg_scmd_ieee()
2866 		sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2872 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd_ieee()
2879 		    ioc->max_sges_in_chain_message) ? sges_left :  in _base_build_sg_scmd_ieee()
2880 		    ioc->max_sges_in_chain_message;  in _base_build_sg_scmd_ieee()
2883 		chain_length = sges_in_segment * ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2885 			chain_length += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2898 			sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2903 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd_ieee()
2923 		sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2940 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,  in _base_build_sg_ieee()  argument
2947 		_base_build_zero_len_sge_ieee(ioc, psge);  in _base_build_sg_ieee()
2959 		psge += ioc->sge_size_ieee;  in _base_build_sg_ieee()
2990 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)  in _base_config_dma_addressing()  argument
2995 	if (ioc->is_mcpu_endpoint || sizeof(dma_addr_t) == 4 ||  in _base_config_dma_addressing()
2997 		ioc->dma_mask = 32;  in _base_config_dma_addressing()
3000 	} else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) {  in _base_config_dma_addressing()
3001 		ioc->dma_mask = 63;  in _base_config_dma_addressing()
3004 		ioc->dma_mask = 64;  in _base_config_dma_addressing()
3008 	if (ioc->use_32bit_dma)  in _base_config_dma_addressing()
3015 	if (ioc->dma_mask > 32) {  in _base_config_dma_addressing()
3016 		ioc->base_add_sg_single = &_base_add_sg_single_64;  in _base_config_dma_addressing()
3017 		ioc->sge_size = sizeof(Mpi2SGESimple64_t);  in _base_config_dma_addressing()
3019 		ioc->base_add_sg_single = &_base_add_sg_single_32;  in _base_config_dma_addressing()
3020 		ioc->sge_size = sizeof(Mpi2SGESimple32_t);  in _base_config_dma_addressing()
3024 	ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",  in _base_config_dma_addressing()
3025 		ioc->dma_mask, convert_to_kb(s.totalram));  in _base_config_dma_addressing()
3038 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)  in _base_check_enable_msix()  argument
3046 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&  in _base_check_enable_msix()
3047 	    ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {  in _base_check_enable_msix()
3051 	base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);  in _base_check_enable_msix()
3053 		dfailprintk(ioc, ioc_info(ioc, "msix not supported\n"));  in _base_check_enable_msix()
3059 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||  in _base_check_enable_msix()
3060 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||  in _base_check_enable_msix()
3061 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||  in _base_check_enable_msix()
3062 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||  in _base_check_enable_msix()
3063 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||  in _base_check_enable_msix()
3064 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||  in _base_check_enable_msix()
3065 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)  in _base_check_enable_msix()
3066 		ioc->msix_vector_count = 1;  in _base_check_enable_msix()
3068 		pci_read_config_word(ioc->pdev, base + 2, &message_control);  in _base_check_enable_msix()
3069 		ioc->msix_vector_count = (message_control & 0x3FF) + 1;  in _base_check_enable_msix()
3071 	dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n",  in _base_check_enable_msix()
3072 				  ioc->msix_vector_count));  in _base_check_enable_msix()
3083 mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_free_irq()  argument
3088 	if (list_empty(&ioc->reply_queue_list))  in mpt3sas_base_free_irq()
3091 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {  in mpt3sas_base_free_irq()
3098 		if (ioc->smp_affinity_enable) {  in mpt3sas_base_free_irq()
3099 			irq = pci_irq_vector(ioc->pdev, reply_q->msix_index);  in mpt3sas_base_free_irq()
3102 		free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),  in mpt3sas_base_free_irq()
3116 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)  in _base_request_irq()  argument
3118 	struct pci_dev *pdev = ioc->pdev;  in _base_request_irq()
3124 		ioc_err(ioc, "unable to allocate memory %zu!\n",  in _base_request_irq()
3128 	reply_q->ioc = ioc;  in _base_request_irq()
3133 	if (index >= ioc->iopoll_q_start_index) {  in _base_request_irq()
3134 		qid = index - ioc->iopoll_q_start_index;  in _base_request_irq()
3136 		    ioc->driver_name, ioc->id, qid);  in _base_request_irq()
3138 		ioc->io_uring_poll_queues[qid].reply_q = reply_q;  in _base_request_irq()
3143 	if (ioc->msix_enable)  in _base_request_irq()
3145 		    ioc->driver_name, ioc->id, index);  in _base_request_irq()
3148 		    ioc->driver_name, ioc->id);  in _base_request_irq()
3159 	list_add_tail(&reply_q->list, &ioc->reply_queue_list);  in _base_request_irq()
3170 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)  in _base_assign_reply_queues()  argument
3174 	int iopoll_q_count = ioc->reply_queue_count -  in _base_assign_reply_queues()
3175 	    ioc->iopoll_q_start_index;  in _base_assign_reply_queues()
3178 	if (!_base_is_controller_msix_enabled(ioc))  in _base_assign_reply_queues()
3181 	if (ioc->msix_load_balance)  in _base_assign_reply_queues()
3184 	memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);  in _base_assign_reply_queues()
3187 	nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,  in _base_assign_reply_queues()
3188 					       ioc->facts.MaxMSIxVectors);  in _base_assign_reply_queues()
3192 	if (ioc->smp_affinity_enable) {  in _base_assign_reply_queues()
3198 		if (ioc->high_iops_queues) {  in _base_assign_reply_queues()
3199 			mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev));  in _base_assign_reply_queues()
3200 			for (index = 0; index < ioc->high_iops_queues;  in _base_assign_reply_queues()
3202 				irq = pci_irq_vector(ioc->pdev, index);  in _base_assign_reply_queues()
3207 		list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_assign_reply_queues()
3210 			if (reply_q->msix_index < ioc->high_iops_queues ||  in _base_assign_reply_queues()
3211 			    reply_q->msix_index >= ioc->iopoll_q_start_index)  in _base_assign_reply_queues()
3214 			mask = pci_irq_get_affinity(ioc->pdev,  in _base_assign_reply_queues()
3217 				ioc_warn(ioc, "no affinity for msi %x\n",  in _base_assign_reply_queues()
3223 				if (cpu >= ioc->cpu_msix_table_sz)  in _base_assign_reply_queues()
3225 				ioc->cpu_msix_table[cpu] = reply_q->msix_index;  in _base_assign_reply_queues()
3233 	nr_msix -= (ioc->high_iops_queues - iopoll_q_count);  in _base_assign_reply_queues()
3236 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_assign_reply_queues()
3239 		if (reply_q->msix_index < ioc->high_iops_queues ||  in _base_assign_reply_queues()
3240 		    reply_q->msix_index >= ioc->iopoll_q_start_index)  in _base_assign_reply_queues()
3250 			ioc->cpu_msix_table[cpu] = reply_q->msix_index;  in _base_assign_reply_queues()
3272 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc,  in _base_check_and_enable_high_iops_queues()  argument
3282 	    ioc->io_uring_poll_queues) {  in _base_check_and_enable_high_iops_queues()
3283 		ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3289 		pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta);  in _base_check_and_enable_high_iops_queues()
3293 			ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3298 	if (!reset_devices && ioc->is_aero_ioc &&  in _base_check_and_enable_high_iops_queues()
3302 		ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES;  in _base_check_and_enable_high_iops_queues()
3304 		ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3313 mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_disable_msix()  argument
3315 	if (!ioc->msix_enable)  in mpt3sas_base_disable_msix()
3317 	pci_free_irq_vectors(ioc->pdev);  in mpt3sas_base_disable_msix()
3318 	ioc->msix_enable = 0;  in mpt3sas_base_disable_msix()
3319 	kfree(ioc->io_uring_poll_queues);  in mpt3sas_base_disable_msix()
3328 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc)  in _base_alloc_irq_vectors()  argument
3331 	struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues };  in _base_alloc_irq_vectors()
3337 	int nr_msix_vectors = ioc->iopoll_q_start_index;  in _base_alloc_irq_vectors()
3340 	if (ioc->smp_affinity_enable)  in _base_alloc_irq_vectors()
3345 	ioc_info(ioc, " %d %d %d\n", ioc->high_iops_queues,  in _base_alloc_irq_vectors()
3346 	    ioc->reply_queue_count, nr_msix_vectors);  in _base_alloc_irq_vectors()
3348 	i = pci_alloc_irq_vectors_affinity(ioc->pdev,  in _base_alloc_irq_vectors()
3349 	    ioc->high_iops_queues,  in _base_alloc_irq_vectors()
3361 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)  in _base_enable_msix()  argument
3368 	ioc->msix_load_balance = false;  in _base_enable_msix()
3376 	if (_base_check_enable_msix(ioc) != 0)  in _base_enable_msix()
3379 	ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count);  in _base_enable_msix()
3381 		ioc->cpu_count, max_msix_vectors);  in _base_enable_msix()
3383 	ioc->reply_queue_count =  in _base_enable_msix()
3384 		min_t(int, ioc->cpu_count, ioc->msix_vector_count);  in _base_enable_msix()
3386 	if (!ioc->rdpq_array_enable && max_msix_vectors == -1)  in _base_enable_msix()
3398 	if (!ioc->combined_reply_queue &&  in _base_enable_msix()
3399 	    ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_enable_msix()
3400 		ioc_info(ioc,  in _base_enable_msix()
3402 		ioc->msix_load_balance = true;  in _base_enable_msix()
3409 	if (ioc->msix_load_balance)  in _base_enable_msix()
3410 		ioc->smp_affinity_enable = 0;  in _base_enable_msix()
3412 	if (!ioc->smp_affinity_enable || ioc->reply_queue_count <= 1)  in _base_enable_msix()
3413 		ioc->shost->host_tagset = 0;  in _base_enable_msix()
3418 	if (ioc->shost->host_tagset)  in _base_enable_msix()
3422 		ioc->io_uring_poll_queues = kcalloc(iopoll_q_count,  in _base_enable_msix()
3424 		if (!ioc->io_uring_poll_queues)  in _base_enable_msix()
3428 	if (ioc->is_aero_ioc)  in _base_enable_msix()
3429 		_base_check_and_enable_high_iops_queues(ioc,  in _base_enable_msix()
3430 		    ioc->msix_vector_count);  in _base_enable_msix()
3436 	ioc->reply_queue_count = min_t(int,  in _base_enable_msix()
3437 	    ioc->reply_queue_count + ioc->high_iops_queues,  in _base_enable_msix()
3438 	    ioc->msix_vector_count);  in _base_enable_msix()
3445 		ioc->reply_queue_count = min_t(int, local_max_msix_vectors,  in _base_enable_msix()
3446 		    ioc->reply_queue_count);  in _base_enable_msix()
3452 		if (ioc->reply_queue_count < (iopoll_q_count + MPT3_MIN_IRQS))  in _base_enable_msix()
3454 		ioc->reply_queue_count = min_t(int,  in _base_enable_msix()
3455 		    ioc->reply_queue_count + iopoll_q_count,  in _base_enable_msix()
3456 		    ioc->msix_vector_count);  in _base_enable_msix()
3462 	ioc->iopoll_q_start_index =  in _base_enable_msix()
3463 	    ioc->reply_queue_count - iopoll_q_count;  in _base_enable_msix()
3465 	r = _base_alloc_irq_vectors(ioc);  in _base_enable_msix()
3467 		ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r);  in _base_enable_msix()
3476 	if (r < ioc->iopoll_q_start_index) {  in _base_enable_msix()
3477 		ioc->reply_queue_count = r + iopoll_q_count;  in _base_enable_msix()
3478 		ioc->iopoll_q_start_index =  in _base_enable_msix()
3479 		    ioc->reply_queue_count - iopoll_q_count;  in _base_enable_msix()
3482 	ioc->msix_enable = 1;  in _base_enable_msix()
3483 	for (i = 0; i < ioc->reply_queue_count; i++) {  in _base_enable_msix()
3484 		r = _base_request_irq(ioc, i);  in _base_enable_msix()
3486 			mpt3sas_base_free_irq(ioc);  in _base_enable_msix()
3487 			mpt3sas_base_disable_msix(ioc);  in _base_enable_msix()
3492 	ioc_info(ioc, "High IOPs queues : %s\n",  in _base_enable_msix()
3493 			ioc->high_iops_queues ? "enabled" : "disabled");  in _base_enable_msix()
3499 	ioc->high_iops_queues = 0;  in _base_enable_msix()
3500 	ioc_info(ioc, "High IOPs queues : disabled\n");  in _base_enable_msix()
3501 	ioc->reply_queue_count = 1;  in _base_enable_msix()
3502 	ioc->iopoll_q_start_index = ioc->reply_queue_count - 0;  in _base_enable_msix()
3503 	r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);  in _base_enable_msix()
3505 		dfailprintk(ioc,  in _base_enable_msix()
3506 			    ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",  in _base_enable_msix()
3509 		r = _base_request_irq(ioc, 0);  in _base_enable_msix()
3519 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_unmap_resources()  argument
3521 	struct pci_dev *pdev = ioc->pdev;  in mpt3sas_base_unmap_resources()
3523 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_unmap_resources()
3525 	mpt3sas_base_free_irq(ioc);  in mpt3sas_base_unmap_resources()
3526 	mpt3sas_base_disable_msix(ioc);  in mpt3sas_base_unmap_resources()
3528 	kfree(ioc->replyPostRegisterIndex);  in mpt3sas_base_unmap_resources()
3529 	ioc->replyPostRegisterIndex = NULL;  in mpt3sas_base_unmap_resources()
3532 	if (ioc->chip_phys) {  in mpt3sas_base_unmap_resources()
3533 		iounmap(ioc->chip);  in mpt3sas_base_unmap_resources()
3534 		ioc->chip_phys = 0;  in mpt3sas_base_unmap_resources()
3538 		pci_release_selected_regions(ioc->pdev, ioc->bars);  in mpt3sas_base_unmap_resources()
3545 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
3555 mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_check_for_fault_and_issue_reset()  argument
3560 	dinitprintk(ioc, pr_info("%s\n", __func__));  in mpt3sas_base_check_for_fault_and_issue_reset()
3561 	if (ioc->pci_error_recovery)  in mpt3sas_base_check_for_fault_and_issue_reset()
3563 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_check_for_fault_and_issue_reset()
3564 	dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state));  in mpt3sas_base_check_for_fault_and_issue_reset()
3567 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_base_check_for_fault_and_issue_reset()
3569 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3570 		rc = _base_diag_reset(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3573 		mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_base_check_for_fault_and_issue_reset()
3575 		mpt3sas_base_wait_for_coredump_completion(ioc, __func__);  in mpt3sas_base_check_for_fault_and_issue_reset()
3576 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3577 		rc = _base_diag_reset(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3590 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_map_resources()  argument
3592 	struct pci_dev *pdev = ioc->pdev;  in mpt3sas_base_map_resources()
3601 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_map_resources()
3603 	ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpt3sas_base_map_resources()
3605 		ioc_warn(ioc, "pci_enable_device_mem: failed\n");  in mpt3sas_base_map_resources()
3606 		ioc->bars = 0;  in mpt3sas_base_map_resources()
3611 	if (pci_request_selected_regions(pdev, ioc->bars,  in mpt3sas_base_map_resources()
3612 	    ioc->driver_name)) {  in mpt3sas_base_map_resources()
3613 		ioc_warn(ioc, "pci_request_selected_regions: failed\n");  in mpt3sas_base_map_resources()
3614 		ioc->bars = 0;  in mpt3sas_base_map_resources()
3625 	if (_base_config_dma_addressing(ioc, pdev) != 0) {  in mpt3sas_base_map_resources()
3626 		ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev));  in mpt3sas_base_map_resources()
3641 			ioc->chip_phys = pci_resource_start(pdev, i);  in mpt3sas_base_map_resources()
3642 			chip_phys = ioc->chip_phys;  in mpt3sas_base_map_resources()
3644 			ioc->chip = ioremap(ioc->chip_phys, memap_sz);  in mpt3sas_base_map_resources()
3648 	if (ioc->chip == NULL) {  in mpt3sas_base_map_resources()
3649 		ioc_err(ioc,  in mpt3sas_base_map_resources()
3655 	mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_map_resources()
3657 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_map_resources()
3659 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_map_resources()
3660 		if (rc || (_base_get_ioc_facts(ioc)))  in mpt3sas_base_map_resources()
3664 	if (!ioc->rdpq_array_enable_assigned) {  in mpt3sas_base_map_resources()
3665 		ioc->rdpq_array_enable = ioc->rdpq_array_capable;  in mpt3sas_base_map_resources()
3666 		ioc->rdpq_array_enable_assigned = 1;  in mpt3sas_base_map_resources()
3669 	r = _base_enable_msix(ioc);  in mpt3sas_base_map_resources()
3673 	iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_map_resources()
3675 		atomic_set(&ioc->io_uring_poll_queues[i].busy, 0);  in mpt3sas_base_map_resources()
3676 		atomic_set(&ioc->io_uring_poll_queues[i].pause, 0);  in mpt3sas_base_map_resources()
3679 	if (!ioc->is_driver_loading)  in mpt3sas_base_map_resources()
3680 		_base_init_irqpolls(ioc);  in mpt3sas_base_map_resources()
3684 	if (ioc->combined_reply_queue) {  in mpt3sas_base_map_resources()
3691 		ioc->replyPostRegisterIndex = kcalloc(  in mpt3sas_base_map_resources()
3692 		     ioc->combined_reply_index_count,  in mpt3sas_base_map_resources()
3694 		if (!ioc->replyPostRegisterIndex) {  in mpt3sas_base_map_resources()
3695 			ioc_err(ioc,  in mpt3sas_base_map_resources()
3701 		for (i = 0; i < ioc->combined_reply_index_count; i++) {  in mpt3sas_base_map_resources()
3702 			ioc->replyPostRegisterIndex[i] =  in mpt3sas_base_map_resources()
3704 				((u8 __force *)&ioc->chip->Doorbell +  in mpt3sas_base_map_resources()
3710 	if (ioc->is_warpdrive) {  in mpt3sas_base_map_resources()
3711 		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)  in mpt3sas_base_map_resources()
3712 		    &ioc->chip->ReplyPostHostIndex;  in mpt3sas_base_map_resources()
3714 		for (i = 1; i < ioc->cpu_msix_table_sz; i++)  in mpt3sas_base_map_resources()
3715 			ioc->reply_post_host_index[i] =  in mpt3sas_base_map_resources()
3717 			((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)  in mpt3sas_base_map_resources()
3721 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in mpt3sas_base_map_resources()
3722 		if (reply_q->msix_index >= ioc->iopoll_q_start_index) {  in mpt3sas_base_map_resources()
3730 			ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC",  in mpt3sas_base_map_resources()
3731 			pci_irq_vector(ioc->pdev, reply_q->msix_index));  in mpt3sas_base_map_resources()
3734 	ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n",  in mpt3sas_base_map_resources()
3735 		 &chip_phys, ioc->chip, memap_sz);  in mpt3sas_base_map_resources()
3736 	ioc_info(ioc, "ioport(0x%016llx), size(%d)\n",  in mpt3sas_base_map_resources()
3744 	mpt3sas_base_unmap_resources(ioc);  in mpt3sas_base_map_resources()
3756 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_msg_frame()  argument
3758 	return (void *)(ioc->request + (smid * ioc->request_sz));  in mpt3sas_base_get_msg_frame()
3769 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_sense_buffer()  argument
3771 	return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));  in mpt3sas_base_get_sense_buffer()
3782 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_sense_buffer_dma()  argument
3784 	return cpu_to_le32(ioc->sense_dma + ((smid - 1) *  in mpt3sas_base_get_sense_buffer_dma()
3796 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_pcie_sgl()  argument
3798 	return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);  in mpt3sas_base_get_pcie_sgl()
3809 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_pcie_sgl_dma()  argument
3811 	return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;  in mpt3sas_base_get_pcie_sgl_dma()
3822 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)  in mpt3sas_base_get_reply_virt_addr()  argument
3826 	return ioc->reply + (phys_addr - (u32)ioc->reply_dma);  in mpt3sas_base_get_reply_virt_addr()
3839 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc,  in _base_get_msix_index()  argument
3843 	if (ioc->msix_load_balance)  in _base_get_msix_index()
3844 		return ioc->reply_queue_count ?  in _base_get_msix_index()
3846 		    &ioc->total_io_cnt), ioc->reply_queue_count) : 0;  in _base_get_msix_index()
3848 	if (scmd && ioc->shost->nr_hw_queues > 1) {  in _base_get_msix_index()
3852 			ioc->high_iops_queues;  in _base_get_msix_index()
3855 	return ioc->cpu_msix_table[raw_smp_processor_id()];  in _base_get_msix_index()
3869 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc,  in _base_get_high_iops_msix_index()  argument
3880 		    atomic64_add_return(1, &ioc->high_iops_outstanding) /  in _base_get_high_iops_msix_index()
3884 	return _base_get_msix_index(ioc, scmd);  in _base_get_high_iops_msix_index()
3895 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)  in mpt3sas_base_get_smid()  argument
3901 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3902 	if (list_empty(&ioc->internal_free_list)) {  in mpt3sas_base_get_smid()
3903 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3904 		ioc_err(ioc, "%s: smid not available\n", __func__);  in mpt3sas_base_get_smid()
3908 	request = list_entry(ioc->internal_free_list.next,  in mpt3sas_base_get_smid()
3913 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3926 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,  in mpt3sas_base_get_smid_scsiio()  argument
3946 	ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag);  in mpt3sas_base_get_smid_scsiio()
3964 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)  in mpt3sas_base_get_smid_hpr()  argument
3970 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
3971 	if (list_empty(&ioc->hpr_free_list)) {  in mpt3sas_base_get_smid_hpr()
3972 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
3976 	request = list_entry(ioc->hpr_free_list.next,  in mpt3sas_base_get_smid_hpr()
3981 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
3986 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc)  in _base_recovery_check()  argument
3991 	if (ioc->shost_recovery && ioc->pending_io_count) {  in _base_recovery_check()
3992 		ioc->pending_io_count = scsi_host_busy(ioc->shost);  in _base_recovery_check()
3993 		if (ioc->pending_io_count == 0)  in _base_recovery_check()
3994 			wake_up(&ioc->reset_wq);  in _base_recovery_check()
3998 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_clear_st()  argument
4006 	atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);  in mpt3sas_base_clear_st()
4016 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_free_smid()  argument
4021 	if (smid < ioc->hi_priority_smid) {  in mpt3sas_base_free_smid()
4025 		st = _get_st_from_smid(ioc, smid);  in mpt3sas_base_free_smid()
4027 			_base_recovery_check(ioc);  in mpt3sas_base_free_smid()
4032 		request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_free_smid()
4033 		memset(request, 0, ioc->request_sz);  in mpt3sas_base_free_smid()
4035 		mpt3sas_base_clear_st(ioc, st);  in mpt3sas_base_free_smid()
4036 		_base_recovery_check(ioc);  in mpt3sas_base_free_smid()
4037 		ioc->io_queue_num[smid - 1] = 0;  in mpt3sas_base_free_smid()
4041 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_free_smid()
4042 	if (smid < ioc->internal_smid) {  in mpt3sas_base_free_smid()
4044 		i = smid - ioc->hi_priority_smid;  in mpt3sas_base_free_smid()
4045 		ioc->hpr_lookup[i].cb_idx = 0xFF;  in mpt3sas_base_free_smid()
4046 		list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);  in mpt3sas_base_free_smid()
4047 	} else if (smid <= ioc->hba_queue_depth) {  in mpt3sas_base_free_smid()
4049 		i = smid - ioc->internal_smid;  in mpt3sas_base_free_smid()
4050 		ioc->internal_lookup[i].cb_idx = 0xFF;  in mpt3sas_base_free_smid()
4051 		list_add(&ioc->internal_lookup[i].tracker_list,  in mpt3sas_base_free_smid()
4052 		    &ioc->internal_free_list);  in mpt3sas_base_free_smid()
4054 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_free_smid()
4114 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_set_and_get_msix_index()  argument
4118 	if (smid < ioc->hi_priority_smid)  in _base_set_and_get_msix_index()
4119 		st = _get_st_from_smid(ioc, smid);  in _base_set_and_get_msix_index()
4122 		return  _base_get_msix_index(ioc, NULL);  in _base_set_and_get_msix_index()
4124 	st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd);  in _base_set_and_get_msix_index()
4135 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc,  in _base_put_smid_mpi_ep_scsi_io()  argument
4141 	__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_mpi_ep_scsi_io()
4143 	_clone_sg_entries(ioc, (void *) mfp, smid);  in _base_put_smid_mpi_ep_scsi_io()
4144 	mpi_req_iomem = (void __force *)ioc->chip +  in _base_put_smid_mpi_ep_scsi_io()
4145 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);  in _base_put_smid_mpi_ep_scsi_io()
4147 					ioc->request_sz);  in _base_put_smid_mpi_ep_scsi_io()
4149 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_mpi_ep_scsi_io()
4153 	_base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_mpi_ep_scsi_io()
4154 	    &ioc->scsi_lookup_lock);  in _base_put_smid_mpi_ep_scsi_io()
4164 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)  in _base_put_smid_scsi_io()  argument
4171 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_scsi_io()
4175 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_scsi_io()
4176 	    &ioc->scsi_lookup_lock);  in _base_put_smid_scsi_io()
4186 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_fast_path()  argument
4194 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_fast_path()
4198 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_fast_path()
4199 	    &ioc->scsi_lookup_lock);  in _base_put_smid_fast_path()
4209 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_hi_priority()  argument
4216 	if (ioc->is_mcpu_endpoint) {  in _base_put_smid_hi_priority()
4217 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_hi_priority()
4220 		mpi_req_iomem = (void __force *)ioc->chip  in _base_put_smid_hi_priority()
4222 					+ (smid * ioc->request_sz);  in _base_put_smid_hi_priority()
4224 							ioc->request_sz);  in _base_put_smid_hi_priority()
4235 	if (ioc->is_mcpu_endpoint)  in _base_put_smid_hi_priority()
4237 				&ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_hi_priority()
4238 				&ioc->scsi_lookup_lock);  in _base_put_smid_hi_priority()
4240 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_hi_priority()
4241 		    &ioc->scsi_lookup_lock);  in _base_put_smid_hi_priority()
4251 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_put_smid_nvme_encap()  argument
4258 	descriptor.Default.MSIxIndex =  _base_set_and_get_msix_index(ioc, smid);  in mpt3sas_base_put_smid_nvme_encap()
4262 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in mpt3sas_base_put_smid_nvme_encap()
4263 	    &ioc->scsi_lookup_lock);  in mpt3sas_base_put_smid_nvme_encap()
4272 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_put_smid_default()  argument
4278 	if (ioc->is_mcpu_endpoint) {  in _base_put_smid_default()
4279 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_default()
4281 		_clone_sg_entries(ioc, (void *) mfp, smid);  in _base_put_smid_default()
4283 		mpi_req_iomem = (void __force *)ioc->chip +  in _base_put_smid_default()
4284 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);  in _base_put_smid_default()
4286 							ioc->request_sz);  in _base_put_smid_default()
4290 	descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_default()
4294 	if (ioc->is_mcpu_endpoint)  in _base_put_smid_default()
4296 				&ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_default()
4297 				&ioc->scsi_lookup_lock);  in _base_put_smid_default()
4299 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_default()
4300 				&ioc->scsi_lookup_lock);  in _base_put_smid_default()
4313 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_scsi_io_atomic()  argument
4320 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_scsi_io_atomic()
4323 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_scsi_io_atomic()
4335 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_fast_path_atomic()  argument
4342 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_fast_path_atomic()
4345 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_fast_path_atomic()
4358 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_hi_priority_atomic()  argument
4368 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_hi_priority_atomic()
4380 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_put_smid_default_atomic()  argument
4386 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_default_atomic()
4389 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_default_atomic()
4397 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)  in _base_display_OEMs_branding()  argument
4399 	if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)  in _base_display_OEMs_branding()
4402 	switch (ioc->pdev->subsystem_vendor) {  in _base_display_OEMs_branding()
4404 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4406 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4408 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4412 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4416 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4420 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4421 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4426 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4428 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4432 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4436 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4440 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4444 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4448 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4452 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4456 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4457 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4462 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4464 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4469 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4473 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4477 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4481 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4482 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4487 			ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4488 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4493 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4495 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4497 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4501 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4505 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4509 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4513 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4517 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4521 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4525 				ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4526 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4531 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4533 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4537 				ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4538 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4543 			ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4544 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4549 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4551 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4553 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4557 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4561 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4565 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4566 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4571 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4573 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4577 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4581 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4582 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4587 			ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4588 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4593 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4595 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4597 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4601 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4602 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4607 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4609 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4613 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4617 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4621 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4625 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4626 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4631 			ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4632 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4649 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)  in _base_display_fwpkg_version()  argument
4662 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_display_fwpkg_version()
4664 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_display_fwpkg_version()
4665 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_display_fwpkg_version()
4670 	fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,  in _base_display_fwpkg_version()
4673 		ioc_err(ioc,  in _base_display_fwpkg_version()
4679 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_display_fwpkg_version()
4681 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_display_fwpkg_version()
4686 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in _base_display_fwpkg_version()
4687 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_display_fwpkg_version()
4688 	ioc->base_cmds.smid = smid;  in _base_display_fwpkg_version()
4693 	ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,  in _base_display_fwpkg_version()
4695 	init_completion(&ioc->base_cmds.done);  in _base_display_fwpkg_version()
4696 	ioc->put_smid_default(ioc, smid);  in _base_display_fwpkg_version()
4698 	wait_for_completion_timeout(&ioc->base_cmds.done,  in _base_display_fwpkg_version()
4700 	ioc_info(ioc, "%s: complete\n", __func__);  in _base_display_fwpkg_version()
4701 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_display_fwpkg_version()
4702 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_display_fwpkg_version()
4708 		if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {  in _base_display_fwpkg_version()
4709 			memcpy(&mpi_reply, ioc->base_cmds.reply,  in _base_display_fwpkg_version()
4728 					ioc_info(ioc,  in _base_display_fwpkg_version()
4740 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in _base_display_fwpkg_version()
4743 		dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data,  in _base_display_fwpkg_version()
4746 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)  in _base_display_fwpkg_version()
4748 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))  in _base_display_fwpkg_version()
4760 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)  in _base_display_ioc_capabilities()  argument
4767 	bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);  in _base_display_ioc_capabilities()
4768 	strncpy(desc, ioc->manu_pg0.ChipName, 16);  in _base_display_ioc_capabilities()
4769 …ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02…  in _base_display_ioc_capabilities()
4771 		 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,  in _base_display_ioc_capabilities()
4772 		 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,  in _base_display_ioc_capabilities()
4773 		 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,  in _base_display_ioc_capabilities()
4774 		 ioc->facts.FWVersion.Word & 0x000000FF,  in _base_display_ioc_capabilities()
4775 		 ioc->pdev->revision,  in _base_display_ioc_capabilities()
4781 	_base_display_OEMs_branding(ioc);  in _base_display_ioc_capabilities()
4783 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {  in _base_display_ioc_capabilities()
4788 	ioc_info(ioc, "Protocol=(");  in _base_display_ioc_capabilities()
4790 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {  in _base_display_ioc_capabilities()
4795 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {  in _base_display_ioc_capabilities()
4803 	if (!ioc->hide_ir_msg) {  in _base_display_ioc_capabilities()
4804 		if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4811 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {  in _base_display_ioc_capabilities()
4816 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {  in _base_display_ioc_capabilities()
4821 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4827 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {  in _base_display_ioc_capabilities()
4832 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4838 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4844 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4850 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4856 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);  in _base_display_ioc_capabilities()
4876 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_update_missing_delay()  argument
4887 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in mpt3sas_base_update_missing_delay()
4895 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4899 	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in mpt3sas_base_update_missing_delay()
4901 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4908 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4933 	if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,  in mpt3sas_base_update_missing_delay()
4941 		ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n",  in mpt3sas_base_update_missing_delay()
4943 		ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n",  in mpt3sas_base_update_missing_delay()
4946 		ioc->device_missing_delay = dmd_new;  in mpt3sas_base_update_missing_delay()
4947 		ioc->io_missing_delay = io_missing_delay;  in mpt3sas_base_update_missing_delay()
4963 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc)  in _base_update_ioc_page1_inlinewith_perf_mode()  argument
4969 	rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy);  in _base_update_ioc_page1_inlinewith_perf_mode()
4972 	memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t));  in _base_update_ioc_page1_inlinewith_perf_mode()
4977 		if (ioc->high_iops_queues) {  in _base_update_ioc_page1_inlinewith_perf_mode()
4978 			ioc_info(ioc,  in _base_update_ioc_page1_inlinewith_perf_mode()
4993 			rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
4996 			ioc_info(ioc, "performance mode: balanced\n");  in _base_update_ioc_page1_inlinewith_perf_mode()
5008 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
5011 		ioc_info(ioc, "performance mode: latency\n");  in _base_update_ioc_page1_inlinewith_perf_mode()
5017 		ioc_info(ioc,  in _base_update_ioc_page1_inlinewith_perf_mode()
5022 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
5038 _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_event_diag_triggers()  argument
5048 	r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply,  in _base_get_event_diag_triggers()
5056 		dinitprintk(ioc,  in _base_get_event_diag_triggers()
5057 		    ioc_err(ioc,  in _base_get_event_diag_triggers()
5066 		ioc->diag_trigger_event.ValidEntries = count;  in _base_get_event_diag_triggers()
5068 		event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0];  in _base_get_event_diag_triggers()
5090 _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_scsi_diag_triggers()  argument
5100 	r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply,  in _base_get_scsi_diag_triggers()
5108 		dinitprintk(ioc,  in _base_get_scsi_diag_triggers()
5109 		    ioc_err(ioc,  in _base_get_scsi_diag_triggers()
5118 		ioc->diag_trigger_scsi.ValidEntries = count;  in _base_get_scsi_diag_triggers()
5120 		scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0];  in _base_get_scsi_diag_triggers()
5142 _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_mpi_diag_triggers()  argument
5152 	r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply,  in _base_get_mpi_diag_triggers()
5160 		dinitprintk(ioc,  in _base_get_mpi_diag_triggers()
5161 		    ioc_err(ioc,  in _base_get_mpi_diag_triggers()
5170 		ioc->diag_trigger_mpi.ValidEntries = count;  in _base_get_mpi_diag_triggers()
5172 		status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0];  in _base_get_mpi_diag_triggers()
5196 _base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_master_diag_triggers()  argument
5203 	r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply,  in _base_get_master_diag_triggers()
5211 		dinitprintk(ioc,  in _base_get_master_diag_triggers()
5212 		    ioc_err(ioc,  in _base_get_master_diag_triggers()
5219 		ioc->diag_trigger_master.MasterData |=  in _base_get_master_diag_triggers()
5238 _base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags)  in _base_check_for_trigger_pages_support()  argument
5245 	r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply,  in _base_check_for_trigger_pages_support()
5268 _base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_diag_triggers()  argument
5276 	ioc->diag_trigger_master.MasterData =  in _base_get_diag_triggers()
5279 	r = _base_check_for_trigger_pages_support(ioc, &trigger_flags);  in _base_get_diag_triggers()
5290 	ioc->supports_trigger_pages = 1;  in _base_get_diag_triggers()
5298 		r = _base_get_master_diag_triggers(ioc);  in _base_get_diag_triggers()
5309 		r = _base_get_event_diag_triggers(ioc);  in _base_get_diag_triggers()
5320 		r = _base_get_scsi_diag_triggers(ioc);  in _base_get_diag_triggers()
5330 		r = _base_get_mpi_diag_triggers(ioc);  in _base_get_diag_triggers()
5346 _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc)  in _base_update_diag_trigger_pages()  argument
5349 	if (ioc->diag_trigger_master.MasterData)  in _base_update_diag_trigger_pages()
5350 		mpt3sas_config_update_driver_trigger_pg1(ioc,  in _base_update_diag_trigger_pages()
5351 		    &ioc->diag_trigger_master, 1);  in _base_update_diag_trigger_pages()
5353 	if (ioc->diag_trigger_event.ValidEntries)  in _base_update_diag_trigger_pages()
5354 		mpt3sas_config_update_driver_trigger_pg2(ioc,  in _base_update_diag_trigger_pages()
5355 		    &ioc->diag_trigger_event, 1);  in _base_update_diag_trigger_pages()
5357 	if (ioc->diag_trigger_scsi.ValidEntries)  in _base_update_diag_trigger_pages()
5358 		mpt3sas_config_update_driver_trigger_pg3(ioc,  in _base_update_diag_trigger_pages()
5359 		    &ioc->diag_trigger_scsi, 1);  in _base_update_diag_trigger_pages()
5361 	if (ioc->diag_trigger_mpi.ValidEntries)  in _base_update_diag_trigger_pages()
5362 		mpt3sas_config_update_driver_trigger_pg4(ioc,  in _base_update_diag_trigger_pages()
5363 		    &ioc->diag_trigger_mpi, 1);  in _base_update_diag_trigger_pages()
5374 static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc)  in _base_assign_fw_reported_qd()  argument
5383 	ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5384 	ioc->max_narrowport_qd = MPT3SAS_SAS_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5385 	ioc->max_sata_qd = MPT3SAS_SATA_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5386 	ioc->max_nvme_qd = MPT3SAS_NVME_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5387 	if (!ioc->is_gen35_ioc)  in _base_assign_fw_reported_qd()
5394 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5397 	rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in _base_assign_fw_reported_qd()
5401 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5406 	ioc->max_wideport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5409 	ioc->max_narrowport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5412 	ioc->max_sata_qd = (depth ? depth : MPT3SAS_SATA_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5415 	rc = mpt3sas_config_get_pcie_iounit_pg1(ioc, &mpi_reply,  in _base_assign_fw_reported_qd()
5419 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5422 	ioc->max_nvme_qd = (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) ?  in _base_assign_fw_reported_qd()
5426 	dinitprintk(ioc, pr_err(  in _base_assign_fw_reported_qd()
5428 	    ioc->max_wideport_qd, ioc->max_narrowport_qd,  in _base_assign_fw_reported_qd()
5429 	    ioc->max_sata_qd, ioc->max_nvme_qd));  in _base_assign_fw_reported_qd()
5442 mpt3sas_atto_validate_nvram(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_atto_validate_nvram()  argument
5460 		ioc_err(ioc, "Invalid ATTO NVRAM checksum\n");  in mpt3sas_atto_validate_nvram()
5470 		ioc_err(ioc, "Invalid ATTO NVRAM signature\n");  in mpt3sas_atto_validate_nvram()
5472 		ioc_info(ioc, "Invalid ATTO NVRAM version");  in mpt3sas_atto_validate_nvram()
5479 		ioc_err(ioc, "Invalid ATTO SAS address\n");  in mpt3sas_atto_validate_nvram()
5493 mpt3sas_atto_get_sas_addr(struct MPT3SAS_ADAPTER *ioc, union ATTO_SAS_ADDRESS *sas_addr)  in mpt3sas_atto_get_sas_addr()  argument
5501 	r = mpt3sas_config_get_manufacturing_pg1(ioc, &mpi_reply, &mfg_pg1);  in mpt3sas_atto_get_sas_addr()
5503 		ioc_err(ioc, "Failed to read manufacturing page 1\n");  in mpt3sas_atto_get_sas_addr()
5509 	r = mpt3sas_atto_validate_nvram(ioc, nvram);  in mpt3sas_atto_get_sas_addr()
5526 mpt3sas_atto_init(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_atto_init()  argument
5537 	r = mpt3sas_atto_get_sas_addr(ioc, &sas_addr);  in mpt3sas_atto_init()
5542 	r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, NULL, 0);  in mpt3sas_atto_init()
5544 		ioc_err(ioc, "Failed to read ATTO bios page 4 header.\n");  in mpt3sas_atto_init()
5551 		ioc_err(ioc, "Failed to allocate memory for ATTO bios page.\n");  in mpt3sas_atto_init()
5556 	r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, bios_pg4, sz);  in mpt3sas_atto_init()
5558 		ioc_err(ioc, "Failed to read ATTO bios page 4\n");  in mpt3sas_atto_init()
5572 	r = mpt3sas_config_set_bios_pg4(ioc, &mpi_reply, bios_pg4, sz);  in mpt3sas_atto_init()
5584 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)  in _base_static_config_pages()  argument
5590 	ioc->nvme_abort_timeout = 30;  in _base_static_config_pages()
5592 	rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply,  in _base_static_config_pages()
5593 	    &ioc->manu_pg0);  in _base_static_config_pages()
5596 	if (ioc->ir_firmware) {  in _base_static_config_pages()
5597 		rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,  in _base_static_config_pages()
5598 		    &ioc->manu_pg10);  in _base_static_config_pages()
5603 	if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) {  in _base_static_config_pages()
5604 		rc = mpt3sas_atto_init(ioc);  in _base_static_config_pages()
5613 	rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply,  in _base_static_config_pages()
5614 	    &ioc->manu_pg11);  in _base_static_config_pages()
5617 	if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) {  in _base_static_config_pages()
5619 		    ioc->name);  in _base_static_config_pages()
5620 		ioc->manu_pg11.EEDPTagMode &= ~0x3;  in _base_static_config_pages()
5621 		ioc->manu_pg11.EEDPTagMode |= 0x1;  in _base_static_config_pages()
5622 		mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,  in _base_static_config_pages()
5623 		    &ioc->manu_pg11);  in _base_static_config_pages()
5625 	if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)  in _base_static_config_pages()
5626 		ioc->tm_custom_handling = 1;  in _base_static_config_pages()
5628 		ioc->tm_custom_handling = 0;  in _base_static_config_pages()
5629 		if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)  in _base_static_config_pages()
5630 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;  in _base_static_config_pages()
5631 		else if (ioc->manu_pg11.NVMeAbortTO >  in _base_static_config_pages()
5633 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;  in _base_static_config_pages()
5635 			ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;  in _base_static_config_pages()
5637 	ioc->time_sync_interval =  in _base_static_config_pages()
5638 	    ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK;  in _base_static_config_pages()
5639 	if (ioc->time_sync_interval) {  in _base_static_config_pages()
5640 		if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK)  in _base_static_config_pages()
5641 			ioc->time_sync_interval =  in _base_static_config_pages()
5642 			    ioc->time_sync_interval * SECONDS_PER_HOUR;  in _base_static_config_pages()
5644 			ioc->time_sync_interval =  in _base_static_config_pages()
5645 			    ioc->time_sync_interval * SECONDS_PER_MIN;  in _base_static_config_pages()
5646 		dinitprintk(ioc, ioc_info(ioc,  in _base_static_config_pages()
5648 		    ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval &  in _base_static_config_pages()
5651 		if (ioc->is_gen35_ioc)  in _base_static_config_pages()
5652 			ioc_warn(ioc,  in _base_static_config_pages()
5655 	rc = _base_assign_fw_reported_qd(ioc);  in _base_static_config_pages()
5662 	if (ioc->pdev->vendor ==  MPI2_MFGPAGE_VENDORID_ATTO)  in _base_static_config_pages()
5663 		ioc->bios_pg3.BiosVersion = 0;  in _base_static_config_pages()
5665 		rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);  in _base_static_config_pages()
5668 		rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);  in _base_static_config_pages()
5673 	rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);  in _base_static_config_pages()
5676 	rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);  in _base_static_config_pages()
5679 	rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);  in _base_static_config_pages()
5682 	rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);  in _base_static_config_pages()
5685 	_base_display_ioc_capabilities(ioc);  in _base_static_config_pages()
5691 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);  in _base_static_config_pages()
5692 	if ((ioc->facts.IOCCapabilities &  in _base_static_config_pages()
5699 	ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);  in _base_static_config_pages()
5700 	rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);  in _base_static_config_pages()
5704 	if (ioc->iounit_pg8.NumSensors)  in _base_static_config_pages()
5705 		ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;  in _base_static_config_pages()
5706 	if (ioc->is_aero_ioc) {  in _base_static_config_pages()
5707 		rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc);  in _base_static_config_pages()
5711 	if (ioc->is_gen35_ioc) {  in _base_static_config_pages()
5712 		if (ioc->is_driver_loading) {  in _base_static_config_pages()
5713 			rc = _base_get_diag_triggers(ioc);  in _base_static_config_pages()
5728 			_base_check_for_trigger_pages_support(ioc, &tg_flags);  in _base_static_config_pages()
5729 			if (!ioc->supports_trigger_pages && tg_flags != -EFAULT)  in _base_static_config_pages()
5730 				_base_update_diag_trigger_pages(ioc);  in _base_static_config_pages()
5731 			else if (ioc->supports_trigger_pages &&  in _base_static_config_pages()
5733 				ioc->supports_trigger_pages = 0;  in _base_static_config_pages()
5746 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_free_enclosure_list()  argument
5752 			enclosure_dev_next, &ioc->enclosure_list, list) {  in mpt3sas_free_enclosure_list()
5765 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)  in _base_release_memory_pools()  argument
5771 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;  in _base_release_memory_pools()
5773 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_release_memory_pools()
5775 	if (ioc->request) {  in _base_release_memory_pools()
5776 		dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz,  in _base_release_memory_pools()
5777 		    ioc->request,  ioc->request_dma);  in _base_release_memory_pools()
5778 		dexitprintk(ioc,  in _base_release_memory_pools()
5779 			    ioc_info(ioc, "request_pool(0x%p): free\n",  in _base_release_memory_pools()
5780 				     ioc->request));  in _base_release_memory_pools()
5781 		ioc->request = NULL;  in _base_release_memory_pools()
5784 	if (ioc->sense) {  in _base_release_memory_pools()
5785 		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);  in _base_release_memory_pools()
5786 		dma_pool_destroy(ioc->sense_dma_pool);  in _base_release_memory_pools()
5787 		dexitprintk(ioc,  in _base_release_memory_pools()
5788 			    ioc_info(ioc, "sense_pool(0x%p): free\n",  in _base_release_memory_pools()
5789 				     ioc->sense));  in _base_release_memory_pools()
5790 		ioc->sense = NULL;  in _base_release_memory_pools()
5793 	if (ioc->reply) {  in _base_release_memory_pools()
5794 		dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);  in _base_release_memory_pools()
5795 		dma_pool_destroy(ioc->reply_dma_pool);  in _base_release_memory_pools()
5796 		dexitprintk(ioc,  in _base_release_memory_pools()
5797 			    ioc_info(ioc, "reply_pool(0x%p): free\n",  in _base_release_memory_pools()
5798 				     ioc->reply));  in _base_release_memory_pools()
5799 		ioc->reply = NULL;  in _base_release_memory_pools()
5802 	if (ioc->reply_free) {  in _base_release_memory_pools()
5803 		dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,  in _base_release_memory_pools()
5804 		    ioc->reply_free_dma);  in _base_release_memory_pools()
5805 		dma_pool_destroy(ioc->reply_free_dma_pool);  in _base_release_memory_pools()
5806 		dexitprintk(ioc,  in _base_release_memory_pools()
5807 			    ioc_info(ioc, "reply_free_pool(0x%p): free\n",  in _base_release_memory_pools()
5808 				     ioc->reply_free));  in _base_release_memory_pools()
5809 		ioc->reply_free = NULL;  in _base_release_memory_pools()
5812 	if (ioc->reply_post) {  in _base_release_memory_pools()
5818 				if (ioc->reply_post[i].reply_post_free) {  in _base_release_memory_pools()
5820 					    ioc->reply_post_free_dma_pool,  in _base_release_memory_pools()
5821 					    ioc->reply_post[i].reply_post_free,  in _base_release_memory_pools()
5822 					ioc->reply_post[i].reply_post_free_dma);  in _base_release_memory_pools()
5823 					dexitprintk(ioc, ioc_info(ioc,  in _base_release_memory_pools()
5825 					   ioc->reply_post[i].reply_post_free));  in _base_release_memory_pools()
5826 					ioc->reply_post[i].reply_post_free =  in _base_release_memory_pools()
5832 		dma_pool_destroy(ioc->reply_post_free_dma_pool);  in _base_release_memory_pools()
5833 		if (ioc->reply_post_free_array &&  in _base_release_memory_pools()
5834 			ioc->rdpq_array_enable) {  in _base_release_memory_pools()
5835 			dma_pool_free(ioc->reply_post_free_array_dma_pool,  in _base_release_memory_pools()
5836 			    ioc->reply_post_free_array,  in _base_release_memory_pools()
5837 			    ioc->reply_post_free_array_dma);  in _base_release_memory_pools()
5838 			ioc->reply_post_free_array = NULL;  in _base_release_memory_pools()
5840 		dma_pool_destroy(ioc->reply_post_free_array_dma_pool);  in _base_release_memory_pools()
5841 		kfree(ioc->reply_post);  in _base_release_memory_pools()
5844 	if (ioc->pcie_sgl_dma_pool) {  in _base_release_memory_pools()
5845 		for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_release_memory_pools()
5846 			dma_pool_free(ioc->pcie_sgl_dma_pool,  in _base_release_memory_pools()
5847 					ioc->pcie_sg_lookup[i].pcie_sgl,  in _base_release_memory_pools()
5848 					ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_release_memory_pools()
5849 			ioc->pcie_sg_lookup[i].pcie_sgl = NULL;  in _base_release_memory_pools()
5851 		dma_pool_destroy(ioc->pcie_sgl_dma_pool);  in _base_release_memory_pools()
5853 	if (ioc->config_page) {  in _base_release_memory_pools()
5854 		dexitprintk(ioc,  in _base_release_memory_pools()
5855 			    ioc_info(ioc, "config_page(0x%p): free\n",  in _base_release_memory_pools()
5856 				     ioc->config_page));  in _base_release_memory_pools()
5857 		dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz,  in _base_release_memory_pools()
5858 		    ioc->config_page, ioc->config_page_dma);  in _base_release_memory_pools()
5861 	kfree(ioc->hpr_lookup);  in _base_release_memory_pools()
5862 	ioc->hpr_lookup = NULL;  in _base_release_memory_pools()
5863 	kfree(ioc->internal_lookup);  in _base_release_memory_pools()
5864 	ioc->internal_lookup = NULL;  in _base_release_memory_pools()
5865 	if (ioc->chain_lookup) {  in _base_release_memory_pools()
5866 		for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_release_memory_pools()
5867 			for (j = ioc->chains_per_prp_buffer;  in _base_release_memory_pools()
5868 			    j < ioc->chains_needed_per_io; j++) {  in _base_release_memory_pools()
5869 				ct = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_release_memory_pools()
5871 					dma_pool_free(ioc->chain_dma_pool,  in _base_release_memory_pools()
5875 			kfree(ioc->chain_lookup[i].chains_per_smid);  in _base_release_memory_pools()
5877 		dma_pool_destroy(ioc->chain_dma_pool);  in _base_release_memory_pools()
5878 		kfree(ioc->chain_lookup);  in _base_release_memory_pools()
5879 		ioc->chain_lookup = NULL;  in _base_release_memory_pools()
5882 	kfree(ioc->io_queue_num);  in _base_release_memory_pools()
5883 	ioc->io_queue_num = NULL;  in _base_release_memory_pools()
5915 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc)  in _base_reduce_hba_queue_depth()  argument
5919 	if ((ioc->hba_queue_depth - reduce_sz) >  in _base_reduce_hba_queue_depth()
5920 	    (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) {  in _base_reduce_hba_queue_depth()
5921 		ioc->hba_queue_depth -= reduce_sz;  in _base_reduce_hba_queue_depth()
5937 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_pcie_sgl_pool()  argument
5942 	ioc->pcie_sgl_dma_pool =  in _base_allocate_pcie_sgl_pool()
5943 	    dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz,  in _base_allocate_pcie_sgl_pool()
5944 	    ioc->page_size, 0);  in _base_allocate_pcie_sgl_pool()
5945 	if (!ioc->pcie_sgl_dma_pool) {  in _base_allocate_pcie_sgl_pool()
5946 		ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n");  in _base_allocate_pcie_sgl_pool()
5950 	ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;  in _base_allocate_pcie_sgl_pool()
5951 	ioc->chains_per_prp_buffer =  in _base_allocate_pcie_sgl_pool()
5952 	    min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io);  in _base_allocate_pcie_sgl_pool()
5953 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_pcie_sgl_pool()
5954 		ioc->pcie_sg_lookup[i].pcie_sgl =  in _base_allocate_pcie_sgl_pool()
5955 		    dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL,  in _base_allocate_pcie_sgl_pool()
5956 		    &ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_allocate_pcie_sgl_pool()
5957 		if (!ioc->pcie_sg_lookup[i].pcie_sgl) {  in _base_allocate_pcie_sgl_pool()
5958 			ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n");  in _base_allocate_pcie_sgl_pool()
5963 		    ioc->pcie_sg_lookup[i].pcie_sgl_dma, sz)) {  in _base_allocate_pcie_sgl_pool()
5964 			ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n",  in _base_allocate_pcie_sgl_pool()
5965 			    ioc->pcie_sg_lookup[i].pcie_sgl,  in _base_allocate_pcie_sgl_pool()
5967 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_allocate_pcie_sgl_pool()
5968 			ioc->use_32bit_dma = true;  in _base_allocate_pcie_sgl_pool()
5972 		for (j = 0; j < ioc->chains_per_prp_buffer; j++) {  in _base_allocate_pcie_sgl_pool()
5973 			ct = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_allocate_pcie_sgl_pool()
5975 			    ioc->pcie_sg_lookup[i].pcie_sgl +  in _base_allocate_pcie_sgl_pool()
5976 			    (j * ioc->chain_segment_sz);  in _base_allocate_pcie_sgl_pool()
5978 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma +  in _base_allocate_pcie_sgl_pool()
5979 			    (j * ioc->chain_segment_sz);  in _base_allocate_pcie_sgl_pool()
5982 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_pcie_sgl_pool()
5984 	    ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));  in _base_allocate_pcie_sgl_pool()
5985 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_pcie_sgl_pool()
5987 	    ioc->chains_per_prp_buffer));  in _base_allocate_pcie_sgl_pool()
6000 _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_chain_dma_pool()  argument
6005 	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,  in _base_allocate_chain_dma_pool()
6006 	    ioc->chain_segment_sz, 16, 0);  in _base_allocate_chain_dma_pool()
6007 	if (!ioc->chain_dma_pool)  in _base_allocate_chain_dma_pool()
6010 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_chain_dma_pool()
6011 		for (j = ioc->chains_per_prp_buffer;  in _base_allocate_chain_dma_pool()
6012 		    j < ioc->chains_needed_per_io; j++) {  in _base_allocate_chain_dma_pool()
6013 			ctr = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_allocate_chain_dma_pool()
6014 			ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool,  in _base_allocate_chain_dma_pool()
6019 			    ctr->chain_buffer_dma, ioc->chain_segment_sz)) {  in _base_allocate_chain_dma_pool()
6020 				ioc_err(ioc,  in _base_allocate_chain_dma_pool()
6024 				ioc->use_32bit_dma = true;  in _base_allocate_chain_dma_pool()
6029 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_chain_dma_pool()
6031 	    ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth *  in _base_allocate_chain_dma_pool()
6032 	    (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) *  in _base_allocate_chain_dma_pool()
6033 	    ioc->chain_segment_sz))/1024));  in _base_allocate_chain_dma_pool()
6045 _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_sense_dma_pool()  argument
6047 	ioc->sense_dma_pool =  in _base_allocate_sense_dma_pool()
6048 	    dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0);  in _base_allocate_sense_dma_pool()
6049 	if (!ioc->sense_dma_pool)  in _base_allocate_sense_dma_pool()
6051 	ioc->sense = dma_pool_alloc(ioc->sense_dma_pool,  in _base_allocate_sense_dma_pool()
6052 	    GFP_KERNEL, &ioc->sense_dma);  in _base_allocate_sense_dma_pool()
6053 	if (!ioc->sense)  in _base_allocate_sense_dma_pool()
6055 	if (!mpt3sas_check_same_4gb_region(ioc->sense_dma, sz)) {  in _base_allocate_sense_dma_pool()
6056 		dinitprintk(ioc, pr_err(  in _base_allocate_sense_dma_pool()
6058 		    ioc->sense, (unsigned long long) ioc->sense_dma));  in _base_allocate_sense_dma_pool()
6059 		ioc->use_32bit_dma = true;  in _base_allocate_sense_dma_pool()
6062 	ioc_info(ioc,  in _base_allocate_sense_dma_pool()
6064 	    ioc->sense, (unsigned long long)ioc->sense_dma,  in _base_allocate_sense_dma_pool()
6065 	    ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024);  in _base_allocate_sense_dma_pool()
6077 _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_reply_pool()  argument
6080 	ioc->reply_dma_pool = dma_pool_create("reply pool",  in _base_allocate_reply_pool()
6081 	    &ioc->pdev->dev, sz, 4, 0);  in _base_allocate_reply_pool()
6082 	if (!ioc->reply_dma_pool)  in _base_allocate_reply_pool()
6084 	ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,  in _base_allocate_reply_pool()
6085 	    &ioc->reply_dma);  in _base_allocate_reply_pool()
6086 	if (!ioc->reply)  in _base_allocate_reply_pool()
6088 	if (!mpt3sas_check_same_4gb_region(ioc->reply_dma, sz)) {  in _base_allocate_reply_pool()
6089 		dinitprintk(ioc, pr_err(  in _base_allocate_reply_pool()
6091 		    ioc->reply, (unsigned long long) ioc->reply_dma));  in _base_allocate_reply_pool()
6092 		ioc->use_32bit_dma = true;  in _base_allocate_reply_pool()
6095 	ioc->reply_dma_min_address = (u32)(ioc->reply_dma);  in _base_allocate_reply_pool()
6096 	ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;  in _base_allocate_reply_pool()
6097 	ioc_info(ioc,  in _base_allocate_reply_pool()
6099 	    ioc->reply, (unsigned long long)ioc->reply_dma,  in _base_allocate_reply_pool()
6100 	    ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024);  in _base_allocate_reply_pool()
6112 _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_reply_free_dma_pool()  argument
6115 	ioc->reply_free_dma_pool = dma_pool_create(  in _base_allocate_reply_free_dma_pool()
6116 	    "reply_free pool", &ioc->pdev->dev, sz, 16, 0);  in _base_allocate_reply_free_dma_pool()
6117 	if (!ioc->reply_free_dma_pool)  in _base_allocate_reply_free_dma_pool()
6119 	ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool,  in _base_allocate_reply_free_dma_pool()
6120 	    GFP_KERNEL, &ioc->reply_free_dma);  in _base_allocate_reply_free_dma_pool()
6121 	if (!ioc->reply_free)  in _base_allocate_reply_free_dma_pool()
6123 	if (!mpt3sas_check_same_4gb_region(ioc->reply_free_dma, sz)) {  in _base_allocate_reply_free_dma_pool()
6124 		dinitprintk(ioc,  in _base_allocate_reply_free_dma_pool()
6126 		    ioc->reply_free, (unsigned long long) ioc->reply_free_dma));  in _base_allocate_reply_free_dma_pool()
6127 		ioc->use_32bit_dma = true;  in _base_allocate_reply_free_dma_pool()
6130 	memset(ioc->reply_free, 0, sz);  in _base_allocate_reply_free_dma_pool()
6131 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_reply_free_dma_pool()
6133 	    ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));  in _base_allocate_reply_free_dma_pool()
6134 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_reply_free_dma_pool()
6136 	    (unsigned long long)ioc->reply_free_dma));  in _base_allocate_reply_free_dma_pool()
6149 _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc,  in _base_allocate_reply_post_free_array()  argument
6152 	ioc->reply_post_free_array_dma_pool =  in _base_allocate_reply_post_free_array()
6154 	    &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);  in _base_allocate_reply_post_free_array()
6155 	if (!ioc->reply_post_free_array_dma_pool)  in _base_allocate_reply_post_free_array()
6157 	ioc->reply_post_free_array =  in _base_allocate_reply_post_free_array()
6158 	    dma_pool_alloc(ioc->reply_post_free_array_dma_pool,  in _base_allocate_reply_post_free_array()
6159 	    GFP_KERNEL, &ioc->reply_post_free_array_dma);  in _base_allocate_reply_post_free_array()
6160 	if (!ioc->reply_post_free_array)  in _base_allocate_reply_post_free_array()
6162 	if (!mpt3sas_check_same_4gb_region(ioc->reply_post_free_array_dma,  in _base_allocate_reply_post_free_array()
6164 		dinitprintk(ioc, pr_err(  in _base_allocate_reply_post_free_array()
6166 		    ioc->reply_free,  in _base_allocate_reply_post_free_array()
6167 		    (unsigned long long) ioc->reply_free_dma));  in _base_allocate_reply_post_free_array()
6168 		ioc->use_32bit_dma = true;  in _base_allocate_reply_post_free_array()
6181 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)  in base_alloc_rdpq_dma_pool()  argument
6185 	int reply_post_free_sz = ioc->reply_post_queue_depth *  in base_alloc_rdpq_dma_pool()
6187 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;  in base_alloc_rdpq_dma_pool()
6189 	ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct),  in base_alloc_rdpq_dma_pool()
6191 	if (!ioc->reply_post)  in base_alloc_rdpq_dma_pool()
6204 	ioc->reply_post_free_dma_pool =  in base_alloc_rdpq_dma_pool()
6206 		    &ioc->pdev->dev, sz, 16, 0);  in base_alloc_rdpq_dma_pool()
6207 	if (!ioc->reply_post_free_dma_pool)  in base_alloc_rdpq_dma_pool()
6211 			ioc->reply_post[i].reply_post_free =  in base_alloc_rdpq_dma_pool()
6212 			    dma_pool_zalloc(ioc->reply_post_free_dma_pool,  in base_alloc_rdpq_dma_pool()
6214 				&ioc->reply_post[i].reply_post_free_dma);  in base_alloc_rdpq_dma_pool()
6215 			if (!ioc->reply_post[i].reply_post_free)  in base_alloc_rdpq_dma_pool()
6227 				ioc->reply_post[i].reply_post_free_dma, sz)) {  in base_alloc_rdpq_dma_pool()
6228 				dinitprintk(ioc,  in base_alloc_rdpq_dma_pool()
6229 				    ioc_err(ioc, "bad Replypost free pool(0x%p)"  in base_alloc_rdpq_dma_pool()
6231 				    ioc->reply_post[i].reply_post_free,  in base_alloc_rdpq_dma_pool()
6233 				    ioc->reply_post[i].reply_post_free_dma));  in base_alloc_rdpq_dma_pool()
6239 			ioc->reply_post[i].reply_post_free =  in base_alloc_rdpq_dma_pool()
6241 			    ((long)ioc->reply_post[i-1].reply_post_free  in base_alloc_rdpq_dma_pool()
6243 			ioc->reply_post[i].reply_post_free_dma =  in base_alloc_rdpq_dma_pool()
6245 			    (ioc->reply_post[i-1].reply_post_free_dma +  in base_alloc_rdpq_dma_pool()
6259 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)  in _base_allocate_memory_pools()  argument
6273 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_allocate_memory_pools()
6277 	facts = &ioc->facts;  in _base_allocate_memory_pools()
6283 		if (ioc->hba_mpi_version_belonged == MPI2_VERSION)  in _base_allocate_memory_pools()
6294 	if (ioc->is_mcpu_endpoint)  in _base_allocate_memory_pools()
6295 		ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;  in _base_allocate_memory_pools()
6302 			ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n",  in _base_allocate_memory_pools()
6305 		ioc->shost->sg_tablesize = sg_tablesize;  in _base_allocate_memory_pools()
6308 	ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),  in _base_allocate_memory_pools()
6310 	if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {  in _base_allocate_memory_pools()
6313 			ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n",  in _base_allocate_memory_pools()
6317 		ioc->internal_depth = 10;  in _base_allocate_memory_pools()
6320 	ioc->hi_priority_depth = ioc->internal_depth - (5);  in _base_allocate_memory_pools()
6324 			ioc->internal_depth, facts->RequestCredit);  in _base_allocate_memory_pools()
6329 		    (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));  in _base_allocate_memory_pools()
6338 	ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;  in _base_allocate_memory_pools()
6341 	ioc->request_sz = facts->IOCRequestFrameSize * 4;  in _base_allocate_memory_pools()
6344 	ioc->reply_sz = facts->ReplyFrameSize * 4;  in _base_allocate_memory_pools()
6347 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_allocate_memory_pools()
6349 			ioc->chain_segment_sz =  in _base_allocate_memory_pools()
6354 			ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *  in _base_allocate_memory_pools()
6357 		ioc->chain_segment_sz = ioc->request_sz;  in _base_allocate_memory_pools()
6360 	sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);  in _base_allocate_memory_pools()
6365 	max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -  in _base_allocate_memory_pools()
6367 	ioc->max_sges_in_main_message = max_sge_elements/sge_size;  in _base_allocate_memory_pools()
6370 	max_sge_elements = ioc->chain_segment_sz - sge_size;  in _base_allocate_memory_pools()
6371 	ioc->max_sges_in_chain_message = max_sge_elements/sge_size;  in _base_allocate_memory_pools()
6376 	chains_needed_per_io = ((ioc->shost->sg_tablesize -  in _base_allocate_memory_pools()
6377 	   ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)  in _base_allocate_memory_pools()
6381 		ioc->shost->sg_tablesize = min_t(u16,  in _base_allocate_memory_pools()
6382 		ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message  in _base_allocate_memory_pools()
6383 		* chains_needed_per_io), ioc->shost->sg_tablesize);  in _base_allocate_memory_pools()
6385 	ioc->chains_needed_per_io = chains_needed_per_io;  in _base_allocate_memory_pools()
6388 	ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;  in _base_allocate_memory_pools()
6391 	if (ioc->is_mcpu_endpoint)  in _base_allocate_memory_pools()
6392 		ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;  in _base_allocate_memory_pools()
6395 		ioc->reply_post_queue_depth = ioc->hba_queue_depth +  in _base_allocate_memory_pools()
6396 			ioc->reply_free_queue_depth +  1;  in _base_allocate_memory_pools()
6398 		if (ioc->reply_post_queue_depth % 16)  in _base_allocate_memory_pools()
6399 			ioc->reply_post_queue_depth += 16 -  in _base_allocate_memory_pools()
6400 				(ioc->reply_post_queue_depth % 16);  in _base_allocate_memory_pools()
6403 	if (ioc->reply_post_queue_depth >  in _base_allocate_memory_pools()
6405 		ioc->reply_post_queue_depth =  in _base_allocate_memory_pools()
6408 		ioc->hba_queue_depth =  in _base_allocate_memory_pools()
6409 				((ioc->reply_post_queue_depth - 64) / 2) - 1;  in _base_allocate_memory_pools()
6410 		ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;  in _base_allocate_memory_pools()
6413 	ioc_info(ioc,  in _base_allocate_memory_pools()
6416 	    ioc->max_sges_in_main_message,  in _base_allocate_memory_pools()
6417 	    ioc->max_sges_in_chain_message,  in _base_allocate_memory_pools()
6418 	    ioc->shost->sg_tablesize,  in _base_allocate_memory_pools()
6419 	    ioc->chains_needed_per_io);  in _base_allocate_memory_pools()
6422 	reply_post_free_sz = ioc->reply_post_queue_depth *  in _base_allocate_memory_pools()
6425 	if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)  in _base_allocate_memory_pools()
6426 	    || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK))  in _base_allocate_memory_pools()
6427 		rdpq_sz = reply_post_free_sz * ioc->reply_queue_count;  in _base_allocate_memory_pools()
6428 	ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz);  in _base_allocate_memory_pools()
6434 		_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6435 		ioc->use_32bit_dma = true;  in _base_allocate_memory_pools()
6436 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {  in _base_allocate_memory_pools()
6437 			ioc_err(ioc,  in _base_allocate_memory_pools()
6438 			    "32 DMA mask failed %s\n", pci_name(ioc->pdev));  in _base_allocate_memory_pools()
6441 		if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz))  in _base_allocate_memory_pools()
6445 	total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 :  in _base_allocate_memory_pools()
6446 	    DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK));  in _base_allocate_memory_pools()
6447 	ioc->scsiio_depth = ioc->hba_queue_depth -  in _base_allocate_memory_pools()
6448 	    ioc->hi_priority_depth - ioc->internal_depth;  in _base_allocate_memory_pools()
6453 	ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;  in _base_allocate_memory_pools()
6454 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6455 		    ioc_info(ioc, "scsi host: can_queue depth (%d)\n",  in _base_allocate_memory_pools()
6456 			     ioc->shost->can_queue));  in _base_allocate_memory_pools()
6461 	ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6462 	sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);  in _base_allocate_memory_pools()
6465 	sz += (ioc->hi_priority_depth * ioc->request_sz);  in _base_allocate_memory_pools()
6468 	sz += (ioc->internal_depth * ioc->request_sz);  in _base_allocate_memory_pools()
6470 	ioc->request_dma_sz = sz;  in _base_allocate_memory_pools()
6471 	ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz,  in _base_allocate_memory_pools()
6472 			&ioc->request_dma, GFP_KERNEL);  in _base_allocate_memory_pools()
6473 	if (!ioc->request) {  in _base_allocate_memory_pools()
6474 …ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(…  in _base_allocate_memory_pools()
6475 			ioc->hba_queue_depth, ioc->chains_needed_per_io,  in _base_allocate_memory_pools()
6476 			ioc->request_sz, sz / 1024);  in _base_allocate_memory_pools()
6477 		if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)  in _base_allocate_memory_pools()
6480 		ioc->hba_queue_depth -= retry_sz;  in _base_allocate_memory_pools()
6481 		_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6486 …ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz…  in _base_allocate_memory_pools()
6487 			ioc->hba_queue_depth, ioc->chains_needed_per_io,  in _base_allocate_memory_pools()
6488 			ioc->request_sz, sz / 1024);  in _base_allocate_memory_pools()
6491 	ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *  in _base_allocate_memory_pools()
6492 	    ioc->request_sz);  in _base_allocate_memory_pools()
6493 	ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *  in _base_allocate_memory_pools()
6494 	    ioc->request_sz);  in _base_allocate_memory_pools()
6497 	ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *  in _base_allocate_memory_pools()
6498 	    ioc->request_sz);  in _base_allocate_memory_pools()
6499 	ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *  in _base_allocate_memory_pools()
6500 	    ioc->request_sz);  in _base_allocate_memory_pools()
6502 	ioc_info(ioc,  in _base_allocate_memory_pools()
6505 	    ioc->request, (unsigned long long) ioc->request_dma,  in _base_allocate_memory_pools()
6506 	    ioc->hba_queue_depth, ioc->request_sz,  in _base_allocate_memory_pools()
6507 	    (ioc->hba_queue_depth * ioc->request_sz) / 1024);  in _base_allocate_memory_pools()
6511 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6512 		    ioc_info(ioc, "scsiio(0x%p): depth(%d)\n",  in _base_allocate_memory_pools()
6513 			     ioc->request, ioc->scsiio_depth));  in _base_allocate_memory_pools()
6515 	ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);  in _base_allocate_memory_pools()
6516 	sz = ioc->scsiio_depth * sizeof(struct chain_lookup);  in _base_allocate_memory_pools()
6517 	ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6518 	if (!ioc->chain_lookup) {  in _base_allocate_memory_pools()
6519 		ioc_err(ioc, "chain_lookup: __get_free_pages failed\n");  in _base_allocate_memory_pools()
6523 	sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);  in _base_allocate_memory_pools()
6524 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_memory_pools()
6525 		ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6526 		if (!ioc->chain_lookup[i].chains_per_smid) {  in _base_allocate_memory_pools()
6527 			ioc_err(ioc, "chain_lookup: kzalloc failed\n");  in _base_allocate_memory_pools()
6533 	ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,  in _base_allocate_memory_pools()
6535 	if (!ioc->hpr_lookup) {  in _base_allocate_memory_pools()
6536 		ioc_err(ioc, "hpr_lookup: kcalloc failed\n");  in _base_allocate_memory_pools()
6539 	ioc->hi_priority_smid = ioc->scsiio_depth + 1;  in _base_allocate_memory_pools()
6540 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6541 		    ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n",  in _base_allocate_memory_pools()
6542 			     ioc->hi_priority,  in _base_allocate_memory_pools()
6543 			     ioc->hi_priority_depth, ioc->hi_priority_smid));  in _base_allocate_memory_pools()
6546 	ioc->internal_lookup = kcalloc(ioc->internal_depth,  in _base_allocate_memory_pools()
6548 	if (!ioc->internal_lookup) {  in _base_allocate_memory_pools()
6549 		ioc_err(ioc, "internal_lookup: kcalloc failed\n");  in _base_allocate_memory_pools()
6552 	ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;  in _base_allocate_memory_pools()
6553 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6554 		    ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n",  in _base_allocate_memory_pools()
6555 			     ioc->internal,  in _base_allocate_memory_pools()
6556 			     ioc->internal_depth, ioc->internal_smid));  in _base_allocate_memory_pools()
6558 	ioc->io_queue_num = kcalloc(ioc->scsiio_depth,  in _base_allocate_memory_pools()
6560 	if (!ioc->io_queue_num)  in _base_allocate_memory_pools()
6576 	ioc->chains_per_prp_buffer = 0;  in _base_allocate_memory_pools()
6577 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {  in _base_allocate_memory_pools()
6579 			(ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;  in _base_allocate_memory_pools()
6580 		nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);  in _base_allocate_memory_pools()
6583 		sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6584 		ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6585 		if (!ioc->pcie_sg_lookup) {  in _base_allocate_memory_pools()
6586 			ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n");  in _base_allocate_memory_pools()
6589 		sz = nvme_blocks_needed * ioc->page_size;  in _base_allocate_memory_pools()
6590 		rc = _base_allocate_pcie_sgl_pool(ioc, sz);  in _base_allocate_memory_pools()
6595 		total_sz += sz * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6598 	rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz);  in _base_allocate_memory_pools()
6603 	total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io -  in _base_allocate_memory_pools()
6604 		ioc->chains_per_prp_buffer) * ioc->scsiio_depth);  in _base_allocate_memory_pools()
6605 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6606 	    ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",  in _base_allocate_memory_pools()
6607 	    ioc->chain_depth, ioc->chain_segment_sz,  in _base_allocate_memory_pools()
6608 	    (ioc->chain_depth * ioc->chain_segment_sz) / 1024));  in _base_allocate_memory_pools()
6610 	sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;  in _base_allocate_memory_pools()
6611 	rc = _base_allocate_sense_dma_pool(ioc, sense_sz);  in _base_allocate_memory_pools()
6617 	ioc_info(ioc,  in _base_allocate_memory_pools()
6620 	    ioc->sense, (unsigned long long)ioc->sense_dma, ioc->scsiio_depth,  in _base_allocate_memory_pools()
6623 	sz = ioc->reply_free_queue_depth * ioc->reply_sz;  in _base_allocate_memory_pools()
6624 	rc = _base_allocate_reply_pool(ioc, sz);  in _base_allocate_memory_pools()
6632 	sz = ioc->reply_free_queue_depth * 4;  in _base_allocate_memory_pools()
6633 	rc = _base_allocate_reply_free_dma_pool(ioc, sz);  in _base_allocate_memory_pools()
6638 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6639 		    ioc_info(ioc, "reply_free_dma (0x%llx)\n",  in _base_allocate_memory_pools()
6640 			     (unsigned long long)ioc->reply_free_dma));  in _base_allocate_memory_pools()
6642 	if (ioc->rdpq_array_enable) {  in _base_allocate_memory_pools()
6643 		reply_post_free_array_sz = ioc->reply_queue_count *  in _base_allocate_memory_pools()
6645 		rc = _base_allocate_reply_post_free_array(ioc,  in _base_allocate_memory_pools()
6652 	ioc->config_page_sz = 512;  in _base_allocate_memory_pools()
6653 	ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev,  in _base_allocate_memory_pools()
6654 			ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL);  in _base_allocate_memory_pools()
6655 	if (!ioc->config_page) {  in _base_allocate_memory_pools()
6656 		ioc_err(ioc, "config page: dma_pool_alloc failed\n");  in _base_allocate_memory_pools()
6660 	ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n",  in _base_allocate_memory_pools()
6661 	    ioc->config_page, (unsigned long long)ioc->config_page_dma,  in _base_allocate_memory_pools()
6662 	    ioc->config_page_sz);  in _base_allocate_memory_pools()
6663 	total_sz += ioc->config_page_sz;  in _base_allocate_memory_pools()
6665 	ioc_info(ioc, "Allocated physical memory: size(%d kB)\n",  in _base_allocate_memory_pools()
6667 	ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",  in _base_allocate_memory_pools()
6668 		 ioc->shost->can_queue, facts->RequestCredit);  in _base_allocate_memory_pools()
6669 	ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n",  in _base_allocate_memory_pools()
6670 		 ioc->shost->sg_tablesize);  in _base_allocate_memory_pools()
6674 	_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6675 	if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) {  in _base_allocate_memory_pools()
6677 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {  in _base_allocate_memory_pools()
6679 			    pci_name(ioc->pdev));  in _base_allocate_memory_pools()
6682 	} else if (_base_reduce_hba_queue_depth(ioc) != 0)  in _base_allocate_memory_pools()
6699 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)  in mpt3sas_base_get_iocstate()  argument
6703 	s = ioc->base_readl(&ioc->chip->Doorbell);  in mpt3sas_base_get_iocstate()
6717 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)  in _base_wait_on_iocstate()  argument
6725 		current_state = mpt3sas_base_get_iocstate(ioc, 1);  in _base_wait_on_iocstate()
6747 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc)  in _base_dump_reg_set()  argument
6750 	u32 __iomem *reg = (u32 __iomem *)ioc->chip;  in _base_dump_reg_set()
6752 	ioc_info(ioc, "System Register set:\n");  in _base_dump_reg_set()
6769 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_int()  argument
6777 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_wait_for_doorbell_int()
6779 			dhsprintk(ioc,  in _base_wait_for_doorbell_int()
6780 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_int()
6789 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_wait_for_doorbell_int()
6795 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_spin_on_doorbell_int()  argument
6803 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_spin_on_doorbell_int()
6805 			dhsprintk(ioc,  in _base_spin_on_doorbell_int()
6806 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_spin_on_doorbell_int()
6815 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_spin_on_doorbell_int()
6832 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_ack()  argument
6841 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_wait_for_doorbell_ack()
6843 			dhsprintk(ioc,  in _base_wait_for_doorbell_ack()
6844 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_ack()
6848 			doorbell = ioc->base_readl(&ioc->chip->Doorbell);  in _base_wait_for_doorbell_ack()
6851 				mpt3sas_print_fault_code(ioc, doorbell);  in _base_wait_for_doorbell_ack()
6856 				mpt3sas_print_coredump_info(ioc, doorbell);  in _base_wait_for_doorbell_ack()
6867 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_wait_for_doorbell_ack()
6880 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_not_used()  argument
6888 		doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell);  in _base_wait_for_doorbell_not_used()
6890 			dhsprintk(ioc,  in _base_wait_for_doorbell_not_used()
6891 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_not_used()
6900 	ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",  in _base_wait_for_doorbell_not_used()
6914 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)  in _base_send_ioc_reset()  argument
6921 		ioc_err(ioc, "%s: unknown reset_type\n", __func__);  in _base_send_ioc_reset()
6925 	if (!(ioc->facts.IOCCapabilities &  in _base_send_ioc_reset()
6929 	ioc_info(ioc, "sending message unit reset !!\n");  in _base_send_ioc_reset()
6932 	    &ioc->chip->Doorbell);  in _base_send_ioc_reset()
6933 	if ((_base_wait_for_doorbell_ack(ioc, 15))) {  in _base_send_ioc_reset()
6938 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);  in _base_send_ioc_reset()
6940 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_send_ioc_reset()
6947 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in _base_send_ioc_reset()
6948 		spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6954 		    MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 ||  in _base_send_ioc_reset()
6955 		    ioc->fault_reset_work_q == NULL)) {  in _base_send_ioc_reset()
6957 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6958 			mpt3sas_print_coredump_info(ioc, ioc_state);  in _base_send_ioc_reset()
6959 			mpt3sas_base_wait_for_coredump_completion(ioc,  in _base_send_ioc_reset()
6962 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6964 		spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6966 	ioc_info(ioc, "message unit reset: %s\n",  in _base_send_ioc_reset()
6982 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout)  in mpt3sas_wait_for_ioc()  argument
6988 		ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in mpt3sas_wait_for_ioc()
6999 		if (ioc->is_driver_loading)  in mpt3sas_wait_for_ioc()
7003 		ioc_info(ioc, "%s: waiting for operational state(count=%d)\n",  in mpt3sas_wait_for_ioc()
7007 		ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__);  in mpt3sas_wait_for_ioc()
7011 		ioc_info(ioc, "ioc is operational\n");  in mpt3sas_wait_for_ioc()
7027 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,  in _base_handshake_req_reply_wait()  argument
7036 	if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {  in _base_handshake_req_reply_wait()
7037 		ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__);  in _base_handshake_req_reply_wait()
7042 	if (ioc->base_readl(&ioc->chip->HostInterruptStatus) &  in _base_handshake_req_reply_wait()
7044 		writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7049 	    &ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7051 	if ((_base_spin_on_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7052 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7056 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7058 	if ((_base_wait_for_doorbell_ack(ioc, 5))) {  in _base_handshake_req_reply_wait()
7059 		ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7066 		writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7067 		if ((_base_wait_for_doorbell_ack(ioc, 5)))  in _base_handshake_req_reply_wait()
7072 		ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7078 	if ((_base_wait_for_doorbell_int(ioc, timeout))) {  in _base_handshake_req_reply_wait()
7079 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7085 	reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7087 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7088 	if ((_base_wait_for_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7089 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7093 	reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7095 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7098 		if ((_base_wait_for_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7099 			ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7104 			ioc->base_readl(&ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7107 			    ioc->base_readl(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7109 		writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7112 	_base_wait_for_doorbell_int(ioc, 5);  in _base_handshake_req_reply_wait()
7113 	if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {  in _base_handshake_req_reply_wait()
7114 		dhsprintk(ioc,  in _base_handshake_req_reply_wait()
7115 			  ioc_info(ioc, "doorbell is in use (line=%d)\n",  in _base_handshake_req_reply_wait()
7118 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7120 	if (ioc->logging_level & MPT_DEBUG_INIT) {  in _base_handshake_req_reply_wait()
7124 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,  in _base_handshake_req_reply_wait()
7145 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_sas_iounit_control()  argument
7154 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_sas_iounit_control()
7156 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_sas_iounit_control()
7158 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_base_sas_iounit_control()
7159 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);  in mpt3sas_base_sas_iounit_control()
7164 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);  in mpt3sas_base_sas_iounit_control()
7168 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in mpt3sas_base_sas_iounit_control()
7170 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_base_sas_iounit_control()
7176 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_base_sas_iounit_control()
7177 	request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_sas_iounit_control()
7178 	ioc->base_cmds.smid = smid;  in mpt3sas_base_sas_iounit_control()
7182 		ioc->ioc_link_reset_in_progress = 1;  in mpt3sas_base_sas_iounit_control()
7183 	init_completion(&ioc->base_cmds.done);  in mpt3sas_base_sas_iounit_control()
7184 	ioc->put_smid_default(ioc, smid);  in mpt3sas_base_sas_iounit_control()
7185 	wait_for_completion_timeout(&ioc->base_cmds.done,  in mpt3sas_base_sas_iounit_control()
7189 	    ioc->ioc_link_reset_in_progress)  in mpt3sas_base_sas_iounit_control()
7190 		ioc->ioc_link_reset_in_progress = 0;  in mpt3sas_base_sas_iounit_control()
7191 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_base_sas_iounit_control()
7192 		mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status,  in mpt3sas_base_sas_iounit_control()
7197 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)  in mpt3sas_base_sas_iounit_control()
7198 		memcpy(mpi_reply, ioc->base_cmds.reply,  in mpt3sas_base_sas_iounit_control()
7202 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_sas_iounit_control()
7207 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_base_sas_iounit_control()
7208 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_sas_iounit_control()
7211 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_sas_iounit_control()
7227 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_scsi_enclosure_processor()  argument
7235 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_scsi_enclosure_processor()
7237 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_scsi_enclosure_processor()
7239 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_base_scsi_enclosure_processor()
7240 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);  in mpt3sas_base_scsi_enclosure_processor()
7245 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);  in mpt3sas_base_scsi_enclosure_processor()
7249 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in mpt3sas_base_scsi_enclosure_processor()
7251 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_base_scsi_enclosure_processor()
7257 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_base_scsi_enclosure_processor()
7258 	request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_scsi_enclosure_processor()
7259 	ioc->base_cmds.smid = smid;  in mpt3sas_base_scsi_enclosure_processor()
7260 	memset(request, 0, ioc->request_sz);  in mpt3sas_base_scsi_enclosure_processor()
7262 	init_completion(&ioc->base_cmds.done);  in mpt3sas_base_scsi_enclosure_processor()
7263 	ioc->put_smid_default(ioc, smid);  in mpt3sas_base_scsi_enclosure_processor()
7264 	wait_for_completion_timeout(&ioc->base_cmds.done,  in mpt3sas_base_scsi_enclosure_processor()
7266 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_base_scsi_enclosure_processor()
7267 		mpt3sas_check_cmd_timeout(ioc,  in mpt3sas_base_scsi_enclosure_processor()
7268 		    ioc->base_cmds.status, mpi_request,  in mpt3sas_base_scsi_enclosure_processor()
7272 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)  in mpt3sas_base_scsi_enclosure_processor()
7273 		memcpy(mpi_reply, ioc->base_cmds.reply,  in mpt3sas_base_scsi_enclosure_processor()
7277 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_scsi_enclosure_processor()
7282 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_base_scsi_enclosure_processor()
7283 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_scsi_enclosure_processor()
7286 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_scsi_enclosure_processor()
7298 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)  in _base_get_port_facts()  argument
7305 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_get_port_facts()
7312 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,  in _base_get_port_facts()
7316 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_get_port_facts()
7320 	pfacts = &ioc->pfacts[port];  in _base_get_port_facts()
7339 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_iocstate()  argument
7344 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_wait_for_iocstate()
7346 	if (ioc->pci_error_recovery) {  in _base_wait_for_iocstate()
7347 		dfailprintk(ioc,  in _base_wait_for_iocstate()
7348 			    ioc_info(ioc, "%s: host in pci error recovery\n",  in _base_wait_for_iocstate()
7353 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in _base_wait_for_iocstate()
7354 	dhsprintk(ioc,  in _base_wait_for_iocstate()
7355 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",  in _base_wait_for_iocstate()
7363 		dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));  in _base_wait_for_iocstate()
7368 		mpt3sas_print_fault_code(ioc, ioc_state &  in _base_wait_for_iocstate()
7373 		ioc_info(ioc,  in _base_wait_for_iocstate()
7379 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);  in _base_wait_for_iocstate()
7381 		dfailprintk(ioc,  in _base_wait_for_iocstate()
7382 			    ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_wait_for_iocstate()
7388 	rc = _base_diag_reset(ioc);  in _base_wait_for_iocstate()
7399 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)  in _base_get_ioc_facts()  argument
7406 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_get_ioc_facts()
7408 	r = _base_wait_for_iocstate(ioc, 10);  in _base_get_ioc_facts()
7410 		dfailprintk(ioc,  in _base_get_ioc_facts()
7411 			    ioc_info(ioc, "%s: failed getting to correct state\n",  in _base_get_ioc_facts()
7419 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,  in _base_get_ioc_facts()
7423 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_get_ioc_facts()
7427 	facts = &ioc->facts;  in _base_get_ioc_facts()
7438 	if (ioc->msix_enable && (facts->MaxMSIxVectors <=  in _base_get_ioc_facts()
7439 	    MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc)))  in _base_get_ioc_facts()
7440 		ioc->combined_reply_queue = 0;  in _base_get_ioc_facts()
7447 		ioc->ir_firmware = 1;  in _base_get_ioc_facts()
7450 		ioc->rdpq_array_capable = 1;  in _base_get_ioc_facts()
7452 	    && ioc->is_aero_ioc)  in _base_get_ioc_facts()
7453 		ioc->atomic_desc_capable = 1;  in _base_get_ioc_facts()
7457 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_get_ioc_facts()
7463 	ioc->shost->max_id = -1;  in _base_get_ioc_facts()
7476 	ioc->page_size = 1 << facts->CurrentHostPageSize;  in _base_get_ioc_facts()
7477 	if (ioc->page_size == 1) {  in _base_get_ioc_facts()
7478 		ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n");  in _base_get_ioc_facts()
7479 		ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;  in _base_get_ioc_facts()
7481 	dinitprintk(ioc,  in _base_get_ioc_facts()
7482 		    ioc_info(ioc, "CurrentHostPageSize(%d)\n",  in _base_get_ioc_facts()
7485 	dinitprintk(ioc,  in _base_get_ioc_facts()
7486 		    ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n",  in _base_get_ioc_facts()
7488 	dinitprintk(ioc,  in _base_get_ioc_facts()
7489 		    ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n",  in _base_get_ioc_facts()
7502 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)  in _base_send_ioc_init()  argument
7511 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_send_ioc_init()
7518 	mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);  in _base_send_ioc_init()
7522 	if (_base_is_controller_msix_enabled(ioc))  in _base_send_ioc_init()
7523 		mpi_request.HostMSIxVectors = ioc->reply_queue_count;  in _base_send_ioc_init()
7524 	mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);  in _base_send_ioc_init()
7526 	    cpu_to_le16(ioc->reply_post_queue_depth);  in _base_send_ioc_init()
7528 	    cpu_to_le16(ioc->reply_free_queue_depth);  in _base_send_ioc_init()
7531 	    cpu_to_le32((u64)ioc->sense_dma >> 32);  in _base_send_ioc_init()
7533 	    cpu_to_le32((u64)ioc->reply_dma >> 32);  in _base_send_ioc_init()
7535 	    cpu_to_le64((u64)ioc->request_dma);  in _base_send_ioc_init()
7537 	    cpu_to_le64((u64)ioc->reply_free_dma);  in _base_send_ioc_init()
7539 	if (ioc->rdpq_array_enable) {  in _base_send_ioc_init()
7540 		reply_post_free_array_sz = ioc->reply_queue_count *  in _base_send_ioc_init()
7542 		memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);  in _base_send_ioc_init()
7543 		for (i = 0; i < ioc->reply_queue_count; i++)  in _base_send_ioc_init()
7544 			ioc->reply_post_free_array[i].RDPQBaseAddress =  in _base_send_ioc_init()
7546 				(u64)ioc->reply_post[i].reply_post_free_dma);  in _base_send_ioc_init()
7549 		    cpu_to_le64((u64)ioc->reply_post_free_array_dma);  in _base_send_ioc_init()
7552 		    cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);  in _base_send_ioc_init()
7567 	if (ioc->logging_level & MPT_DEBUG_INIT) {  in _base_send_ioc_init()
7572 		ioc_info(ioc, "\toffset:data\n");  in _base_send_ioc_init()
7574 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,  in _base_send_ioc_init()
7578 	r = _base_handshake_req_reply_wait(ioc,  in _base_send_ioc_init()
7583 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_send_ioc_init()
7590 		ioc_err(ioc, "%s: failed\n", __func__);  in _base_send_ioc_init()
7595 	ioc->timestamp_update_count = 0;  in _base_send_ioc_init()
7610 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in mpt3sas_port_enable_done()  argument
7616 	if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)  in mpt3sas_port_enable_done()
7619 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_port_enable_done()
7626 	ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;  in mpt3sas_port_enable_done()
7627 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;  in mpt3sas_port_enable_done()
7628 	ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;  in mpt3sas_port_enable_done()
7629 	memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in mpt3sas_port_enable_done()
7632 		ioc->port_enable_failed = 1;  in mpt3sas_port_enable_done()
7634 	if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) {  in mpt3sas_port_enable_done()
7635 		ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC;  in mpt3sas_port_enable_done()
7637 			mpt3sas_port_enable_complete(ioc);  in mpt3sas_port_enable_done()
7640 			ioc->start_scan_failed = ioc_status;  in mpt3sas_port_enable_done()
7641 			ioc->start_scan = 0;  in mpt3sas_port_enable_done()
7645 	complete(&ioc->port_enable_cmds.done);  in mpt3sas_port_enable_done()
7656 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)  in _base_send_port_enable()  argument
7664 	ioc_info(ioc, "sending port enable !!\n");  in _base_send_port_enable()
7666 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in _base_send_port_enable()
7667 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_send_port_enable()
7671 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);  in _base_send_port_enable()
7673 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_send_port_enable()
7677 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;  in _base_send_port_enable()
7678 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_send_port_enable()
7679 	ioc->port_enable_cmds.smid = smid;  in _base_send_port_enable()
7683 	init_completion(&ioc->port_enable_cmds.done);  in _base_send_port_enable()
7684 	ioc->put_smid_default(ioc, smid);  in _base_send_port_enable()
7685 	wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);  in _base_send_port_enable()
7686 	if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_send_port_enable()
7687 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_send_port_enable()
7690 		if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)  in _base_send_port_enable()
7697 	mpi_reply = ioc->port_enable_cmds.reply;  in _base_send_port_enable()
7700 		ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n",  in _base_send_port_enable()
7707 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in _base_send_port_enable()
7708 	ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED");  in _base_send_port_enable()
7719 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_port_enable()  argument
7724 	ioc_info(ioc, "sending port enable !!\n");  in mpt3sas_port_enable()
7726 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_port_enable()
7727 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in mpt3sas_port_enable()
7731 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);  in mpt3sas_port_enable()
7733 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_port_enable()
7736 	ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED;  in mpt3sas_port_enable()
7737 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_port_enable()
7738 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC;  in mpt3sas_port_enable()
7739 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_port_enable()
7740 	ioc->port_enable_cmds.smid = smid;  in mpt3sas_port_enable()
7744 	ioc->put_smid_default(ioc, smid);  in mpt3sas_port_enable()
7758 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)  in _base_determine_wait_on_discovery()  argument
7766 	if (ioc->ir_firmware)  in _base_determine_wait_on_discovery()
7770 	if (!ioc->bios_pg3.BiosVersion)  in _base_determine_wait_on_discovery()
7780 	if ((ioc->bios_pg2.CurrentBootDeviceForm &  in _base_determine_wait_on_discovery()
7784 	   (ioc->bios_pg2.ReqBootDeviceForm &  in _base_determine_wait_on_discovery()
7788 	   (ioc->bios_pg2.ReqAltBootDeviceForm &  in _base_determine_wait_on_discovery()
7804 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)  in _base_unmask_events()  argument
7814 		ioc->event_masks[0] &= ~desired_event;  in _base_unmask_events()
7816 		ioc->event_masks[1] &= ~desired_event;  in _base_unmask_events()
7818 		ioc->event_masks[2] &= ~desired_event;  in _base_unmask_events()
7820 		ioc->event_masks[3] &= ~desired_event;  in _base_unmask_events()
7830 _base_event_notification(struct MPT3SAS_ADAPTER *ioc)  in _base_event_notification()  argument
7837 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_event_notification()
7839 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_event_notification()
7840 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_event_notification()
7844 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_event_notification()
7846 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_event_notification()
7849 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in _base_event_notification()
7850 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_event_notification()
7851 	ioc->base_cmds.smid = smid;  in _base_event_notification()
7858 		    cpu_to_le32(ioc->event_masks[i]);  in _base_event_notification()
7859 	init_completion(&ioc->base_cmds.done);  in _base_event_notification()
7860 	ioc->put_smid_default(ioc, smid);  in _base_event_notification()
7861 	wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);  in _base_event_notification()
7862 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_event_notification()
7863 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_event_notification()
7866 		if (ioc->base_cmds.status & MPT3_CMD_RESET)  in _base_event_notification()
7872 		dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__));  in _base_event_notification()
7873 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in _base_event_notification()
7876 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)  in _base_event_notification()
7878 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))  in _base_event_notification()
7894 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)  in mpt3sas_base_validate_event_type()  argument
7906 			    (ioc->event_masks[i] & desired_event)) {  in mpt3sas_base_validate_event_type()
7907 				ioc->event_masks[i] &= ~desired_event;  in mpt3sas_base_validate_event_type()
7917 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_validate_event_type()
7918 	_base_event_notification(ioc);  in mpt3sas_base_validate_event_type()
7919 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_validate_event_type()
7929 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc)  in _base_diag_reset()  argument
7936 	ioc_info(ioc, "sending diag reset !!\n");  in _base_diag_reset()
7938 	pci_cfg_access_lock(ioc->pdev);  in _base_diag_reset()
7940 	drsprintk(ioc, ioc_info(ioc, "clear interrupts\n"));  in _base_diag_reset()
7947 		drsprintk(ioc, ioc_info(ioc, "write magic sequence\n"));  in _base_diag_reset()
7948 		writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7949 		writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7950 		writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7951 		writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7952 		writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7953 		writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7954 		writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7960 			ioc_info(ioc,  in _base_diag_reset()
7962 			_base_dump_reg_set(ioc);  in _base_diag_reset()
7966 		host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);  in _base_diag_reset()
7967 		drsprintk(ioc,  in _base_diag_reset()
7968 			  ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",  in _base_diag_reset()
7973 	hcb_size = ioc->base_readl(&ioc->chip->HCBSize);  in _base_diag_reset()
7975 	drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n"));  in _base_diag_reset()
7977 	     &ioc->chip->HostDiagnostic);  in _base_diag_reset()
7986 		host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);  in _base_diag_reset()
7989 			ioc_info(ioc,  in _base_diag_reset()
7991 			_base_dump_reg_set(ioc);  in _base_diag_reset()
8002 		drsprintk(ioc,  in _base_diag_reset()
8003 			  ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n"));  in _base_diag_reset()
8006 		writel(host_diagnostic, &ioc->chip->HostDiagnostic);  in _base_diag_reset()
8008 		drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n"));  in _base_diag_reset()
8010 		    &ioc->chip->HCBSize);  in _base_diag_reset()
8013 	drsprintk(ioc, ioc_info(ioc, "restart the adapter\n"));  in _base_diag_reset()
8015 	    &ioc->chip->HostDiagnostic);  in _base_diag_reset()
8017 	drsprintk(ioc,  in _base_diag_reset()
8018 		  ioc_info(ioc, "disable writes to the diagnostic register\n"));  in _base_diag_reset()
8019 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
8021 	drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n"));  in _base_diag_reset()
8022 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);  in _base_diag_reset()
8024 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_diag_reset()
8026 		_base_dump_reg_set(ioc);  in _base_diag_reset()
8030 	pci_cfg_access_unlock(ioc->pdev);  in _base_diag_reset()
8031 	ioc_info(ioc, "diag reset: SUCCESS\n");  in _base_diag_reset()
8035 	pci_cfg_access_unlock(ioc->pdev);  in _base_diag_reset()
8036 	ioc_err(ioc, "diag reset: FAILED\n");  in _base_diag_reset()
8048 mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)  in mpt3sas_base_make_ioc_ready()  argument
8054 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_make_ioc_ready()
8056 	if (ioc->pci_error_recovery)  in mpt3sas_base_make_ioc_ready()
8059 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_make_ioc_ready()
8060 	dhsprintk(ioc,  in mpt3sas_base_make_ioc_ready()
8061 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",  in mpt3sas_base_make_ioc_ready()
8070 				ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in mpt3sas_base_make_ioc_ready()
8075 			ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_make_ioc_ready()
8083 		ioc_info(ioc, "unexpected doorbell active!\n");  in mpt3sas_base_make_ioc_ready()
8088 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_base_make_ioc_ready()
8101 		if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) {  in mpt3sas_base_make_ioc_ready()
8102 			mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_base_make_ioc_ready()
8104 			mpt3sas_base_wait_for_coredump_completion(ioc,  in mpt3sas_base_make_ioc_ready()
8114 		if (!(_base_send_ioc_reset(ioc,  in mpt3sas_base_make_ioc_ready()
8120 	rc = _base_diag_reset(ioc);  in mpt3sas_base_make_ioc_ready()
8131 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)  in _base_make_ioc_operational()  argument
8144 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_make_ioc_operational()
8148 	    &ioc->delayed_tr_list, list) {  in _base_make_ioc_operational()
8155 	    &ioc->delayed_tr_volume_list, list) {  in _base_make_ioc_operational()
8161 	    &ioc->delayed_sc_list, list) {  in _base_make_ioc_operational()
8167 	    &ioc->delayed_event_ack_list, list) {  in _base_make_ioc_operational()
8172 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _base_make_ioc_operational()
8175 	INIT_LIST_HEAD(&ioc->hpr_free_list);  in _base_make_ioc_operational()
8176 	smid = ioc->hi_priority_smid;  in _base_make_ioc_operational()
8177 	for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {  in _base_make_ioc_operational()
8178 		ioc->hpr_lookup[i].cb_idx = 0xFF;  in _base_make_ioc_operational()
8179 		ioc->hpr_lookup[i].smid = smid;  in _base_make_ioc_operational()
8180 		list_add_tail(&ioc->hpr_lookup[i].tracker_list,  in _base_make_ioc_operational()
8181 		    &ioc->hpr_free_list);  in _base_make_ioc_operational()
8185 	INIT_LIST_HEAD(&ioc->internal_free_list);  in _base_make_ioc_operational()
8186 	smid = ioc->internal_smid;  in _base_make_ioc_operational()
8187 	for (i = 0; i < ioc->internal_depth; i++, smid++) {  in _base_make_ioc_operational()
8188 		ioc->internal_lookup[i].cb_idx = 0xFF;  in _base_make_ioc_operational()
8189 		ioc->internal_lookup[i].smid = smid;  in _base_make_ioc_operational()
8190 		list_add_tail(&ioc->internal_lookup[i].tracker_list,  in _base_make_ioc_operational()
8191 		    &ioc->internal_free_list);  in _base_make_ioc_operational()
8194 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _base_make_ioc_operational()
8197 	for (i = 0, reply_address = (u32)ioc->reply_dma ;  in _base_make_ioc_operational()
8198 	    i < ioc->reply_free_queue_depth ; i++, reply_address +=  in _base_make_ioc_operational()
8199 	    ioc->reply_sz) {  in _base_make_ioc_operational()
8200 		ioc->reply_free[i] = cpu_to_le32(reply_address);  in _base_make_ioc_operational()
8201 		if (ioc->is_mcpu_endpoint)  in _base_make_ioc_operational()
8202 			_base_clone_reply_to_sys_mem(ioc,  in _base_make_ioc_operational()
8207 	if (ioc->is_driver_loading)  in _base_make_ioc_operational()
8208 		_base_assign_reply_queues(ioc);  in _base_make_ioc_operational()
8212 	reply_post_free_contig = ioc->reply_post[0].reply_post_free;  in _base_make_ioc_operational()
8213 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_make_ioc_operational()
8218 		if (ioc->rdpq_array_enable) {  in _base_make_ioc_operational()
8220 				ioc->reply_post[index++].reply_post_free;  in _base_make_ioc_operational()
8223 			reply_post_free_contig += ioc->reply_post_queue_depth;  in _base_make_ioc_operational()
8227 		for (i = 0; i < ioc->reply_post_queue_depth; i++)  in _base_make_ioc_operational()
8230 		if (!_base_is_controller_msix_enabled(ioc))  in _base_make_ioc_operational()
8235 	r = _base_send_ioc_init(ioc);  in _base_make_ioc_operational()
8242 		if (!ioc->is_driver_loading)  in _base_make_ioc_operational()
8245 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in _base_make_ioc_operational()
8246 		if (rc || (_base_send_ioc_init(ioc)))  in _base_make_ioc_operational()
8251 	ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;  in _base_make_ioc_operational()
8252 	writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);  in _base_make_ioc_operational()
8255 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_make_ioc_operational()
8256 		if (ioc->combined_reply_queue)  in _base_make_ioc_operational()
8259 			   ioc->replyPostRegisterIndex[reply_q->msix_index/8]);  in _base_make_ioc_operational()
8263 				&ioc->chip->ReplyPostHostIndex);  in _base_make_ioc_operational()
8265 		if (!_base_is_controller_msix_enabled(ioc))  in _base_make_ioc_operational()
8271 	mpt3sas_base_unmask_interrupts(ioc);  in _base_make_ioc_operational()
8273 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_make_ioc_operational()
8274 		r = _base_display_fwpkg_version(ioc);  in _base_make_ioc_operational()
8279 	r = _base_static_config_pages(ioc);  in _base_make_ioc_operational()
8283 	r = _base_event_notification(ioc);  in _base_make_ioc_operational()
8287 	if (!ioc->shost_recovery) {  in _base_make_ioc_operational()
8289 		if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier  in _base_make_ioc_operational()
8292 			    le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &  in _base_make_ioc_operational()
8295 				ioc->mfg_pg10_hide_flag = hide_flag;  in _base_make_ioc_operational()
8298 		ioc->wait_for_discovery_to_complete =  in _base_make_ioc_operational()
8299 		    _base_determine_wait_on_discovery(ioc);  in _base_make_ioc_operational()
8304 	r = _base_send_port_enable(ioc);  in _base_make_ioc_operational()
8316 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_free_resources()  argument
8318 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_free_resources()
8321 	mutex_lock(&ioc->pci_access_mutex);  in mpt3sas_base_free_resources()
8322 	if (ioc->chip_phys && ioc->chip) {  in mpt3sas_base_free_resources()
8323 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_free_resources()
8324 		ioc->shost_recovery = 1;  in mpt3sas_base_free_resources()
8325 		mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in mpt3sas_base_free_resources()
8326 		ioc->shost_recovery = 0;  in mpt3sas_base_free_resources()
8329 	mpt3sas_base_unmap_resources(ioc);  in mpt3sas_base_free_resources()
8330 	mutex_unlock(&ioc->pci_access_mutex);  in mpt3sas_base_free_resources()
8341 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_attach()  argument
8346 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_attach()
8349 	ioc->cpu_count = num_online_cpus();  in mpt3sas_base_attach()
8352 	ioc->cpu_msix_table_sz = last_cpu_id + 1;  in mpt3sas_base_attach()
8353 	ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8354 	ioc->reply_queue_count = 1;  in mpt3sas_base_attach()
8355 	if (!ioc->cpu_msix_table) {  in mpt3sas_base_attach()
8356 		ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n");  in mpt3sas_base_attach()
8361 	if (ioc->is_warpdrive) {  in mpt3sas_base_attach()
8362 		ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,  in mpt3sas_base_attach()
8364 		if (!ioc->reply_post_host_index) {  in mpt3sas_base_attach()
8365 			ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n");  in mpt3sas_base_attach()
8371 	ioc->smp_affinity_enable = smp_affinity_enable;  in mpt3sas_base_attach()
8373 	ioc->rdpq_array_enable_assigned = 0;  in mpt3sas_base_attach()
8374 	ioc->use_32bit_dma = false;  in mpt3sas_base_attach()
8375 	ioc->dma_mask = 64;  in mpt3sas_base_attach()
8376 	if (ioc->is_aero_ioc)  in mpt3sas_base_attach()
8377 		ioc->base_readl = &_base_readl_aero;  in mpt3sas_base_attach()
8379 		ioc->base_readl = &_base_readl;  in mpt3sas_base_attach()
8380 	r = mpt3sas_base_map_resources(ioc);  in mpt3sas_base_attach()
8384 	pci_set_drvdata(ioc->pdev, ioc->shost);  in mpt3sas_base_attach()
8385 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_attach()
8387 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_attach()
8388 		if (rc || (_base_get_ioc_facts(ioc)))  in mpt3sas_base_attach()
8392 	switch (ioc->hba_mpi_version_belonged) {  in mpt3sas_base_attach()
8394 		ioc->build_sg_scmd = &_base_build_sg_scmd;  in mpt3sas_base_attach()
8395 		ioc->build_sg = &_base_build_sg;  in mpt3sas_base_attach()
8396 		ioc->build_zero_len_sge = &_base_build_zero_len_sge;  in mpt3sas_base_attach()
8397 		ioc->get_msix_index_for_smlio = &_base_get_msix_index;  in mpt3sas_base_attach()
8407 		ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;  in mpt3sas_base_attach()
8408 		ioc->build_sg = &_base_build_sg_ieee;  in mpt3sas_base_attach()
8409 		ioc->build_nvme_prp = &_base_build_nvme_prp;  in mpt3sas_base_attach()
8410 		ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;  in mpt3sas_base_attach()
8411 		ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);  in mpt3sas_base_attach()
8412 		if (ioc->high_iops_queues)  in mpt3sas_base_attach()
8413 			ioc->get_msix_index_for_smlio =  in mpt3sas_base_attach()
8416 			ioc->get_msix_index_for_smlio = &_base_get_msix_index;  in mpt3sas_base_attach()
8419 	if (ioc->atomic_desc_capable) {  in mpt3sas_base_attach()
8420 		ioc->put_smid_default = &_base_put_smid_default_atomic;  in mpt3sas_base_attach()
8421 		ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;  in mpt3sas_base_attach()
8422 		ioc->put_smid_fast_path =  in mpt3sas_base_attach()
8424 		ioc->put_smid_hi_priority =  in mpt3sas_base_attach()
8427 		ioc->put_smid_default = &_base_put_smid_default;  in mpt3sas_base_attach()
8428 		ioc->put_smid_fast_path = &_base_put_smid_fast_path;  in mpt3sas_base_attach()
8429 		ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;  in mpt3sas_base_attach()
8430 		if (ioc->is_mcpu_endpoint)  in mpt3sas_base_attach()
8431 			ioc->put_smid_scsi_io =  in mpt3sas_base_attach()
8434 			ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;  in mpt3sas_base_attach()
8442 	ioc->build_sg_mpi = &_base_build_sg;  in mpt3sas_base_attach()
8443 	ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;  in mpt3sas_base_attach()
8445 	r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in mpt3sas_base_attach()
8449 	ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,  in mpt3sas_base_attach()
8451 	if (!ioc->pfacts) {  in mpt3sas_base_attach()
8456 	for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {  in mpt3sas_base_attach()
8457 		r = _base_get_port_facts(ioc, i);  in mpt3sas_base_attach()
8459 			rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_attach()
8460 			if (rc || (_base_get_port_facts(ioc, i)))  in mpt3sas_base_attach()
8465 	r = _base_allocate_memory_pools(ioc);  in mpt3sas_base_attach()
8470 		ioc->thresh_hold = irqpoll_weight;  in mpt3sas_base_attach()
8472 		ioc->thresh_hold = ioc->hba_queue_depth/4;  in mpt3sas_base_attach()
8474 	_base_init_irqpolls(ioc);  in mpt3sas_base_attach()
8475 	init_waitqueue_head(&ioc->reset_wq);  in mpt3sas_base_attach()
8478 	ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);  in mpt3sas_base_attach()
8479 	if (ioc->facts.MaxDevHandle % 8)  in mpt3sas_base_attach()
8480 		ioc->pd_handles_sz++;  in mpt3sas_base_attach()
8481 	ioc->pd_handles = kzalloc(ioc->pd_handles_sz,  in mpt3sas_base_attach()
8483 	if (!ioc->pd_handles) {  in mpt3sas_base_attach()
8487 	ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,  in mpt3sas_base_attach()
8489 	if (!ioc->blocking_handles) {  in mpt3sas_base_attach()
8495 	ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);  in mpt3sas_base_attach()
8496 	if (ioc->facts.MaxDevHandle % 8)  in mpt3sas_base_attach()
8497 		ioc->pend_os_device_add_sz++;  in mpt3sas_base_attach()
8498 	ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,  in mpt3sas_base_attach()
8500 	if (!ioc->pend_os_device_add) {  in mpt3sas_base_attach()
8505 	ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;  in mpt3sas_base_attach()
8506 	ioc->device_remove_in_progress =  in mpt3sas_base_attach()
8507 		kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8508 	if (!ioc->device_remove_in_progress) {  in mpt3sas_base_attach()
8513 	ioc->fwfault_debug = mpt3sas_fwfault_debug;  in mpt3sas_base_attach()
8516 	mutex_init(&ioc->base_cmds.mutex);  in mpt3sas_base_attach()
8517 	ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8518 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8521 	ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8522 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8525 	ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8526 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8527 	mutex_init(&ioc->transport_cmds.mutex);  in mpt3sas_base_attach()
8530 	ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8531 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8532 	mutex_init(&ioc->scsih_cmds.mutex);  in mpt3sas_base_attach()
8535 	ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8536 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8537 	mutex_init(&ioc->tm_cmds.mutex);  in mpt3sas_base_attach()
8540 	ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8541 	ioc->config_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8542 	mutex_init(&ioc->config_cmds.mutex);  in mpt3sas_base_attach()
8545 	ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8546 	ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);  in mpt3sas_base_attach()
8547 	ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8548 	mutex_init(&ioc->ctl_cmds.mutex);  in mpt3sas_base_attach()
8550 	if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||  in mpt3sas_base_attach()
8551 	    !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||  in mpt3sas_base_attach()
8552 	    !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||  in mpt3sas_base_attach()
8553 	    !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {  in mpt3sas_base_attach()
8559 		ioc->event_masks[i] = -1;  in mpt3sas_base_attach()
8562 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);  in mpt3sas_base_attach()
8563 	_base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);  in mpt3sas_base_attach()
8564 	_base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);  in mpt3sas_base_attach()
8565 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);  in mpt3sas_base_attach()
8566 	_base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);  in mpt3sas_base_attach()
8567 	_base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);  in mpt3sas_base_attach()
8568 	_base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);  in mpt3sas_base_attach()
8569 	_base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);  in mpt3sas_base_attach()
8570 	_base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);  in mpt3sas_base_attach()
8571 	_base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);  in mpt3sas_base_attach()
8572 	_base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);  in mpt3sas_base_attach()
8573 	_base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);  in mpt3sas_base_attach()
8574 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);  in mpt3sas_base_attach()
8575 	if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {  in mpt3sas_base_attach()
8576 		if (ioc->is_gen35_ioc) {  in mpt3sas_base_attach()
8577 			_base_unmask_events(ioc,  in mpt3sas_base_attach()
8579 			_base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);  in mpt3sas_base_attach()
8580 			_base_unmask_events(ioc,  in mpt3sas_base_attach()
8584 	r = _base_make_ioc_operational(ioc);  in mpt3sas_base_attach()
8586 		r = _base_make_ioc_operational(ioc);  in mpt3sas_base_attach()
8595 	memcpy(&ioc->prev_fw_facts, &ioc->facts,  in mpt3sas_base_attach()
8598 	ioc->non_operational_loop = 0;  in mpt3sas_base_attach()
8599 	ioc->ioc_coredump_loop = 0;  in mpt3sas_base_attach()
8600 	ioc->got_task_abort_from_ioctl = 0;  in mpt3sas_base_attach()
8605 	ioc->remove_host = 1;  in mpt3sas_base_attach()
8607 	mpt3sas_base_free_resources(ioc);  in mpt3sas_base_attach()
8608 	_base_release_memory_pools(ioc);  in mpt3sas_base_attach()
8609 	pci_set_drvdata(ioc->pdev, NULL);  in mpt3sas_base_attach()
8610 	kfree(ioc->cpu_msix_table);  in mpt3sas_base_attach()
8611 	if (ioc->is_warpdrive)  in mpt3sas_base_attach()
8612 		kfree(ioc->reply_post_host_index);  in mpt3sas_base_attach()
8613 	kfree(ioc->pd_handles);  in mpt3sas_base_attach()
8614 	kfree(ioc->blocking_handles);  in mpt3sas_base_attach()
8615 	kfree(ioc->device_remove_in_progress);  in mpt3sas_base_attach()
8616 	kfree(ioc->pend_os_device_add);  in mpt3sas_base_attach()
8617 	kfree(ioc->tm_cmds.reply);  in mpt3sas_base_attach()
8618 	kfree(ioc->transport_cmds.reply);  in mpt3sas_base_attach()
8619 	kfree(ioc->scsih_cmds.reply);  in mpt3sas_base_attach()
8620 	kfree(ioc->config_cmds.reply);  in mpt3sas_base_attach()
8621 	kfree(ioc->base_cmds.reply);  in mpt3sas_base_attach()
8622 	kfree(ioc->port_enable_cmds.reply);  in mpt3sas_base_attach()
8623 	kfree(ioc->ctl_cmds.reply);  in mpt3sas_base_attach()
8624 	kfree(ioc->ctl_cmds.sense);  in mpt3sas_base_attach()
8625 	kfree(ioc->pfacts);  in mpt3sas_base_attach()
8626 	ioc->ctl_cmds.reply = NULL;  in mpt3sas_base_attach()
8627 	ioc->base_cmds.reply = NULL;  in mpt3sas_base_attach()
8628 	ioc->tm_cmds.reply = NULL;  in mpt3sas_base_attach()
8629 	ioc->scsih_cmds.reply = NULL;  in mpt3sas_base_attach()
8630 	ioc->transport_cmds.reply = NULL;  in mpt3sas_base_attach()
8631 	ioc->config_cmds.reply = NULL;  in mpt3sas_base_attach()
8632 	ioc->pfacts = NULL;  in mpt3sas_base_attach()
8642 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_detach()  argument
8644 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_detach()
8646 	mpt3sas_base_stop_watchdog(ioc);  in mpt3sas_base_detach()
8647 	mpt3sas_base_free_resources(ioc);  in mpt3sas_base_detach()
8648 	_base_release_memory_pools(ioc);  in mpt3sas_base_detach()
8649 	mpt3sas_free_enclosure_list(ioc);  in mpt3sas_base_detach()
8650 	pci_set_drvdata(ioc->pdev, NULL);  in mpt3sas_base_detach()
8651 	kfree(ioc->cpu_msix_table);  in mpt3sas_base_detach()
8652 	if (ioc->is_warpdrive)  in mpt3sas_base_detach()
8653 		kfree(ioc->reply_post_host_index);  in mpt3sas_base_detach()
8654 	kfree(ioc->pd_handles);  in mpt3sas_base_detach()
8655 	kfree(ioc->blocking_handles);  in mpt3sas_base_detach()
8656 	kfree(ioc->device_remove_in_progress);  in mpt3sas_base_detach()
8657 	kfree(ioc->pend_os_device_add);  in mpt3sas_base_detach()
8658 	kfree(ioc->pfacts);  in mpt3sas_base_detach()
8659 	kfree(ioc->ctl_cmds.reply);  in mpt3sas_base_detach()
8660 	kfree(ioc->ctl_cmds.sense);  in mpt3sas_base_detach()
8661 	kfree(ioc->base_cmds.reply);  in mpt3sas_base_detach()
8662 	kfree(ioc->port_enable_cmds.reply);  in mpt3sas_base_detach()
8663 	kfree(ioc->tm_cmds.reply);  in mpt3sas_base_detach()
8664 	kfree(ioc->transport_cmds.reply);  in mpt3sas_base_detach()
8665 	kfree(ioc->scsih_cmds.reply);  in mpt3sas_base_detach()
8666 	kfree(ioc->config_cmds.reply);  in mpt3sas_base_detach()
8673 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)  in _base_pre_reset_handler()  argument
8675 	mpt3sas_scsih_pre_reset_handler(ioc);  in _base_pre_reset_handler()
8676 	mpt3sas_ctl_pre_reset_handler(ioc);  in _base_pre_reset_handler()
8677 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));  in _base_pre_reset_handler()
8685 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc)  in _base_clear_outstanding_mpt_commands()  argument
8687 	dtmprintk(ioc,  in _base_clear_outstanding_mpt_commands()
8688 	    ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__));  in _base_clear_outstanding_mpt_commands()
8689 	if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8690 		ioc->transport_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8691 		mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8692 		complete(&ioc->transport_cmds.done);  in _base_clear_outstanding_mpt_commands()
8694 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8695 		ioc->base_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8696 		mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8697 		complete(&ioc->base_cmds.done);  in _base_clear_outstanding_mpt_commands()
8699 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8700 		ioc->port_enable_failed = 1;  in _base_clear_outstanding_mpt_commands()
8701 		ioc->port_enable_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8702 		mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8703 		if (ioc->is_driver_loading) {  in _base_clear_outstanding_mpt_commands()
8704 			ioc->start_scan_failed =  in _base_clear_outstanding_mpt_commands()
8706 			ioc->start_scan = 0;  in _base_clear_outstanding_mpt_commands()
8708 			complete(&ioc->port_enable_cmds.done);  in _base_clear_outstanding_mpt_commands()
8711 	if (ioc->config_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8712 		ioc->config_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8713 		mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8714 		ioc->config_cmds.smid = USHRT_MAX;  in _base_clear_outstanding_mpt_commands()
8715 		complete(&ioc->config_cmds.done);  in _base_clear_outstanding_mpt_commands()
8723 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc)  in _base_clear_outstanding_commands()  argument
8725 	mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc);  in _base_clear_outstanding_commands()
8726 	mpt3sas_ctl_clear_outstanding_ioctls(ioc);  in _base_clear_outstanding_commands()
8727 	_base_clear_outstanding_mpt_commands(ioc);  in _base_clear_outstanding_commands()
8734 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)  in _base_reset_done_handler()  argument
8736 	mpt3sas_scsih_reset_done_handler(ioc);  in _base_reset_done_handler()
8737 	mpt3sas_ctl_reset_done_handler(ioc);  in _base_reset_done_handler()
8738 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));  in _base_reset_done_handler()
8749 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_wait_for_commands_to_complete()  argument
8753 	ioc->pending_io_count = 0;  in mpt3sas_wait_for_commands_to_complete()
8755 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_wait_for_commands_to_complete()
8760 	ioc->pending_io_count = scsi_host_busy(ioc->shost);  in mpt3sas_wait_for_commands_to_complete()
8762 	if (!ioc->pending_io_count)  in mpt3sas_wait_for_commands_to_complete()
8766 	wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);  in mpt3sas_wait_for_commands_to_complete()
8777 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc)  in _base_check_ioc_facts_changes()  argument
8782 	struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts;  in _base_check_ioc_facts_changes()
8784 	if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) {  in _base_check_ioc_facts_changes()
8785 		pd_handles_sz = (ioc->facts.MaxDevHandle / 8);  in _base_check_ioc_facts_changes()
8786 		if (ioc->facts.MaxDevHandle % 8)  in _base_check_ioc_facts_changes()
8789 		pd_handles = krealloc(ioc->pd_handles, pd_handles_sz,  in _base_check_ioc_facts_changes()
8792 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8797 		memset(pd_handles + ioc->pd_handles_sz, 0,  in _base_check_ioc_facts_changes()
8798 		    (pd_handles_sz - ioc->pd_handles_sz));  in _base_check_ioc_facts_changes()
8799 		ioc->pd_handles = pd_handles;  in _base_check_ioc_facts_changes()
8801 		blocking_handles = krealloc(ioc->blocking_handles,  in _base_check_ioc_facts_changes()
8804 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8810 		memset(blocking_handles + ioc->pd_handles_sz, 0,  in _base_check_ioc_facts_changes()
8811 		    (pd_handles_sz - ioc->pd_handles_sz));  in _base_check_ioc_facts_changes()
8812 		ioc->blocking_handles = blocking_handles;  in _base_check_ioc_facts_changes()
8813 		ioc->pd_handles_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8815 		pend_os_device_add = krealloc(ioc->pend_os_device_add,  in _base_check_ioc_facts_changes()
8818 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8823 		memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0,  in _base_check_ioc_facts_changes()
8824 		    (pd_handles_sz - ioc->pend_os_device_add_sz));  in _base_check_ioc_facts_changes()
8825 		ioc->pend_os_device_add = pend_os_device_add;  in _base_check_ioc_facts_changes()
8826 		ioc->pend_os_device_add_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8829 		    ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL);  in _base_check_ioc_facts_changes()
8831 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8838 		    ioc->device_remove_in_progress_sz, 0,  in _base_check_ioc_facts_changes()
8839 		    (pd_handles_sz - ioc->device_remove_in_progress_sz));  in _base_check_ioc_facts_changes()
8840 		ioc->device_remove_in_progress = device_remove_in_progress;  in _base_check_ioc_facts_changes()
8841 		ioc->device_remove_in_progress_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8844 	memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts));  in _base_check_ioc_facts_changes()
8856 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_hard_reset_handler()  argument
8864 	dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__));  in mpt3sas_base_hard_reset_handler()
8866 	if (ioc->pci_error_recovery) {  in mpt3sas_base_hard_reset_handler()
8867 		ioc_err(ioc, "%s: pci error recovery reset\n", __func__);  in mpt3sas_base_hard_reset_handler()
8873 		mpt3sas_halt_firmware(ioc);  in mpt3sas_base_hard_reset_handler()
8876 	mutex_lock(&ioc->reset_in_progress_mutex);  in mpt3sas_base_hard_reset_handler()
8878 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8879 	ioc->shost_recovery = 1;  in mpt3sas_base_hard_reset_handler()
8880 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8882 	if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &  in mpt3sas_base_hard_reset_handler()
8884 	    (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &  in mpt3sas_base_hard_reset_handler()
8887 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_hard_reset_handler()
8892 			ioc->htb_rel.trigger_info_dwords[1] =  in mpt3sas_base_hard_reset_handler()
8896 	_base_pre_reset_handler(ioc);  in mpt3sas_base_hard_reset_handler()
8897 	mpt3sas_wait_for_commands_to_complete(ioc);  in mpt3sas_base_hard_reset_handler()
8898 	mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_hard_reset_handler()
8899 	mpt3sas_base_pause_mq_polling(ioc);  in mpt3sas_base_hard_reset_handler()
8900 	r = mpt3sas_base_make_ioc_ready(ioc, type);  in mpt3sas_base_hard_reset_handler()
8903 	_base_clear_outstanding_commands(ioc);  in mpt3sas_base_hard_reset_handler()
8908 	if (ioc->is_driver_loading && ioc->port_enable_failed) {  in mpt3sas_base_hard_reset_handler()
8909 		ioc->remove_host = 1;  in mpt3sas_base_hard_reset_handler()
8913 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_hard_reset_handler()
8917 	r = _base_check_ioc_facts_changes(ioc);  in mpt3sas_base_hard_reset_handler()
8919 		ioc_info(ioc,  in mpt3sas_base_hard_reset_handler()
8924 	if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)  in mpt3sas_base_hard_reset_handler()
8927 		      " firmware version is running\n", ioc->name);  in mpt3sas_base_hard_reset_handler()
8929 	r = _base_make_ioc_operational(ioc);  in mpt3sas_base_hard_reset_handler()
8931 		_base_reset_done_handler(ioc);  in mpt3sas_base_hard_reset_handler()
8934 	ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED");  in mpt3sas_base_hard_reset_handler()
8936 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8937 	ioc->shost_recovery = 0;  in mpt3sas_base_hard_reset_handler()
8938 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8939 	ioc->ioc_reset_count++;  in mpt3sas_base_hard_reset_handler()
8940 	mutex_unlock(&ioc->reset_in_progress_mutex);  in mpt3sas_base_hard_reset_handler()
8941 	mpt3sas_base_resume_mq_polling(ioc);  in mpt3sas_base_hard_reset_handler()
8946 			mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);  in mpt3sas_base_hard_reset_handler()
8948 			mpt3sas_trigger_master(ioc,  in mpt3sas_base_hard_reset_handler()
8951 	dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__));  in mpt3sas_base_hard_reset_handler()