401 static dma_addr_t esp_cur_dma_addr(struct esp_cmd_entry *ent,  in esp_cur_dma_addr()  argument
406 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_cur_dma_addr()
407 		return ent->sense_dma +  in esp_cur_dma_addr()
408 			(ent->sense_ptr - cmd->sense_buffer);  in esp_cur_dma_addr()
416 static unsigned int esp_cur_dma_len(struct esp_cmd_entry *ent,  in esp_cur_dma_len()  argument
421 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_cur_dma_len()
423 			(ent->sense_ptr - cmd->sense_buffer);  in esp_cur_dma_len()
428 static void esp_advance_dma(struct esp *esp, struct esp_cmd_entry *ent,  in esp_advance_dma()  argument
433 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_advance_dma()
434 		ent->sense_ptr += len;  in esp_advance_dma()
462 static void esp_save_pointers(struct esp *esp, struct esp_cmd_entry *ent)  in esp_save_pointers()  argument
464 	struct scsi_cmnd *cmd = ent->cmd;  in esp_save_pointers()
467 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_save_pointers()
468 		ent->saved_sense_ptr = ent->sense_ptr;  in esp_save_pointers()
471 	ent->saved_cur_residue = spriv->cur_residue;  in esp_save_pointers()
472 	ent->saved_prv_sg = spriv->prv_sg;  in esp_save_pointers()
473 	ent->saved_cur_sg = spriv->cur_sg;  in esp_save_pointers()
474 	ent->saved_tot_residue = spriv->tot_residue;  in esp_save_pointers()
477 static void esp_restore_pointers(struct esp *esp, struct esp_cmd_entry *ent)  in esp_restore_pointers()  argument
479 	struct scsi_cmnd *cmd = ent->cmd;  in esp_restore_pointers()
482 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_restore_pointers()
483 		ent->sense_ptr = ent->saved_sense_ptr;  in esp_restore_pointers()
486 	spriv->cur_residue = ent->saved_cur_residue;  in esp_restore_pointers()
487 	spriv->prv_sg = ent->saved_prv_sg;  in esp_restore_pointers()
488 	spriv->cur_sg = ent->saved_cur_sg;  in esp_restore_pointers()
489 	spriv->tot_residue = ent->saved_tot_residue;  in esp_restore_pointers()
571 static int esp_alloc_lun_tag(struct esp_cmd_entry *ent,  in esp_alloc_lun_tag()  argument
574 	if (!ent->orig_tag[0]) {  in esp_alloc_lun_tag()
598 		lp->non_tagged_cmd = ent;  in esp_alloc_lun_tag()
606 	BUG_ON(lp->tagged_cmds[ent->orig_tag[1]]);  in esp_alloc_lun_tag()
608 	lp->tagged_cmds[ent->orig_tag[1]] = ent;  in esp_alloc_lun_tag()
614 static void esp_free_lun_tag(struct esp_cmd_entry *ent,  in esp_free_lun_tag()  argument
617 	if (ent->orig_tag[0]) {  in esp_free_lun_tag()
618 		BUG_ON(lp->tagged_cmds[ent->orig_tag[1]] != ent);  in esp_free_lun_tag()
619 		lp->tagged_cmds[ent->orig_tag[1]] = NULL;  in esp_free_lun_tag()
622 		BUG_ON(lp->non_tagged_cmd != ent);  in esp_free_lun_tag()
627 static void esp_map_sense(struct esp *esp, struct esp_cmd_entry *ent)  in esp_map_sense()  argument
629 	ent->sense_ptr = ent->cmd->sense_buffer;  in esp_map_sense()
631 		ent->sense_dma = (uintptr_t)ent->sense_ptr;  in esp_map_sense()
635 	ent->sense_dma = dma_map_single(esp->dev, ent->sense_ptr,  in esp_map_sense()
639 static void esp_unmap_sense(struct esp *esp, struct esp_cmd_entry *ent)  in esp_unmap_sense()  argument
642 		dma_unmap_single(esp->dev, ent->sense_dma,  in esp_unmap_sense()
644 	ent->sense_ptr = NULL;  in esp_unmap_sense()
654 static void esp_autosense(struct esp *esp, struct esp_cmd_entry *ent)  in esp_autosense()  argument
656 	struct scsi_cmnd *cmd = ent->cmd;  in esp_autosense()
665 	if (!ent->sense_ptr) {  in esp_autosense()
668 		esp_map_sense(esp, ent);  in esp_autosense()
670 	ent->saved_sense_ptr = ent->sense_ptr;  in esp_autosense()
672 	esp->active_cmd = ent;  in esp_autosense()
703 	struct esp_cmd_entry *ent;  in find_and_prep_issuable_command()  local
705 	list_for_each_entry(ent, &esp->queued_cmds, list) {  in find_and_prep_issuable_command()
706 		struct scsi_cmnd *cmd = ent->cmd;  in find_and_prep_issuable_command()
710 		if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in find_and_prep_issuable_command()
711 			ent->tag[0] = 0;  in find_and_prep_issuable_command()
712 			ent->tag[1] = 0;  in find_and_prep_issuable_command()
713 			return ent;  in find_and_prep_issuable_command()
716 		if (!spi_populate_tag_msg(&ent->tag[0], cmd)) {  in find_and_prep_issuable_command()
717 			ent->tag[0] = 0;  in find_and_prep_issuable_command()
718 			ent->tag[1] = 0;  in find_and_prep_issuable_command()
720 		ent->orig_tag[0] = ent->tag[0];  in find_and_prep_issuable_command()
721 		ent->orig_tag[1] = ent->tag[1];  in find_and_prep_issuable_command()
723 		if (esp_alloc_lun_tag(ent, lp) < 0)  in find_and_prep_issuable_command()
726 		return ent;  in find_and_prep_issuable_command()
737 	struct esp_cmd_entry *ent;  in esp_maybe_execute_command()  local
747 	ent = find_and_prep_issuable_command(esp);  in esp_maybe_execute_command()
748 	if (!ent)  in esp_maybe_execute_command()
751 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_maybe_execute_command()
752 		esp_autosense(esp, ent);  in esp_maybe_execute_command()
756 	cmd = ent->cmd;  in esp_maybe_execute_command()
762 	list_move(&ent->list, &esp->active_cmds);  in esp_maybe_execute_command()
764 	esp->active_cmd = ent;  in esp_maybe_execute_command()
767 	esp_save_pointers(esp, ent);  in esp_maybe_execute_command()
819 	if (ent->tag[0] && esp->rev == ESP100) {  in esp_maybe_execute_command()
830 		if (ent->tag[0]) {  in esp_maybe_execute_command()
834 			esp->msg_out[0] = ent->tag[0];  in esp_maybe_execute_command()
835 			esp->msg_out[1] = ent->tag[1];  in esp_maybe_execute_command()
843 		if (ent->tag[0]) {  in esp_maybe_execute_command()
844 			*p++ = ent->tag[0];  in esp_maybe_execute_command()
845 			*p++ = ent->tag[1];  in esp_maybe_execute_command()
890 static void esp_put_ent(struct esp *esp, struct esp_cmd_entry *ent)  in esp_put_ent()  argument
892 	list_add(&ent->list, &esp->esp_cmd_pool);  in esp_put_ent()
895 static void esp_cmd_is_done(struct esp *esp, struct esp_cmd_entry *ent,  in esp_cmd_is_done()  argument
904 	esp_free_lun_tag(ent, dev->hostdata);  in esp_cmd_is_done()
907 	if (ent->eh_done) {  in esp_cmd_is_done()
908 		complete(ent->eh_done);  in esp_cmd_is_done()
909 		ent->eh_done = NULL;  in esp_cmd_is_done()
912 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE) {  in esp_cmd_is_done()
913 		esp_unmap_sense(esp, ent);  in esp_cmd_is_done()
924 		ent->flags &= ~ESP_CMD_FLAG_AUTOSENSE;  in esp_cmd_is_done()
938 	list_del(&ent->list);  in esp_cmd_is_done()
939 	esp_put_ent(esp, ent);  in esp_cmd_is_done()
950 static void esp_event_queue_full(struct esp *esp, struct esp_cmd_entry *ent)  in esp_event_queue_full()  argument
952 	struct scsi_device *dev = ent->cmd->device;  in esp_event_queue_full()
963 	struct esp_cmd_entry *ent;  in esp_queuecommand_lck()  local
965 	ent = esp_get_ent(esp);  in esp_queuecommand_lck()
966 	if (!ent)  in esp_queuecommand_lck()
969 	ent->cmd = cmd;  in esp_queuecommand_lck()
976 	list_add_tail(&ent->list, &esp->queued_cmds);  in esp_queuecommand_lck()
1054 	struct esp_cmd_entry *ent;  in esp_reconnect_with_tag()  local
1131 	ent = lp->tagged_cmds[esp->command_block[1]];  in esp_reconnect_with_tag()
1132 	if (!ent) {  in esp_reconnect_with_tag()
1139 	return ent;  in esp_reconnect_with_tag()
1144 	struct esp_cmd_entry *ent;  in esp_reconnect()  local
1208 	ent = lp->non_tagged_cmd;  in esp_reconnect()
1209 	if (!ent) {  in esp_reconnect()
1210 		ent = esp_reconnect_with_tag(esp, lp);  in esp_reconnect()
1211 		if (!ent)  in esp_reconnect()
1215 	esp->active_cmd = ent;  in esp_reconnect()
1218 	esp_restore_pointers(esp, ent);  in esp_reconnect()
1229 	struct esp_cmd_entry *ent;  in esp_finish_select()  local
1236 	ent = esp->active_cmd;  in esp_finish_select()
1237 	cmd = ent->cmd;  in esp_finish_select()
1244 		esp_cmd_is_done(esp, ent, cmd, (DID_ERROR << 16));  in esp_finish_select()
1257 		if (!(ent->flags & ESP_CMD_FLAG_AUTOSENSE)) {  in esp_finish_select()
1259 			esp_free_lun_tag(ent, cmd->device->hostdata);  in esp_finish_select()
1264 			esp_unmap_sense(esp, ent);  in esp_finish_select()
1270 		list_move(&ent->list, &esp->queued_cmds);  in esp_finish_select()
1289 		esp_cmd_is_done(esp, ent, cmd, (DID_BAD_TARGET << 16));  in esp_finish_select()
1321 static int esp_data_bytes_sent(struct esp *esp, struct esp_cmd_entry *ent,  in esp_data_bytes_sent()  argument
1352 	if (fifo_cnt == 1 && ent->flags & ESP_CMD_FLAG_RESIDUAL) {  in esp_data_bytes_sent()
1357 		if (ent->flags & ESP_CMD_FLAG_AUTOSENSE)  in esp_data_bytes_sent()
1358 			ent->sense_ptr[bytes_sent] = bval;  in esp_data_bytes_sent()
1371 		ent->flags &= ~ESP_CMD_FLAG_RESIDUAL;  in esp_data_bytes_sent()
1373 	if (!(ent->flags & ESP_CMD_FLAG_WRITE))  in esp_data_bytes_sent()
1405 		if (!(ent->flags & ESP_CMD_FLAG_WRITE))  in esp_data_bytes_sent()
1456 	struct esp_cmd_entry *ent = esp->active_cmd;  in esp_msgin_reject()  local
1457 	struct scsi_cmnd *cmd = ent->cmd;  in esp_msgin_reject()
1596 	struct esp_cmd_entry *ent = esp->active_cmd;  in esp_msgin_extended()  local
1597 	struct scsi_cmnd *cmd = ent->cmd;  in esp_msgin_extended()
1644 		struct esp_cmd_entry *ent;  in esp_msgin_process()  local
1652 		ent = esp->active_cmd;  in esp_msgin_process()
1653 		spriv = ESP_CMD_PRIV(ent->cmd);  in esp_msgin_process()
1674 		struct esp_cmd_entry *ent = esp->active_cmd;  in esp_msgin_process()  local
1676 		ent->message = msg0;  in esp_msgin_process()
1744 		struct esp_cmd_entry *ent = esp->active_cmd;  in esp_process_event()  local
1745 		struct scsi_cmnd *cmd = ent->cmd;  in esp_process_event()
1746 		dma_addr_t dma_addr = esp_cur_dma_addr(ent, cmd);  in esp_process_event()
1747 		unsigned int dma_len = esp_cur_dma_len(ent, cmd);  in esp_process_event()
1753 			ent->flags |= ESP_CMD_FLAG_WRITE;  in esp_process_event()
1755 			ent->flags &= ~ESP_CMD_FLAG_WRITE;  in esp_process_event()
1770 				     (unsigned long long)esp_cur_dma_addr(ent, cmd),  in esp_process_event()
1771 				     esp_cur_dma_len(ent, cmd));  in esp_process_event()
1785 		struct esp_cmd_entry *ent = esp->active_cmd;  in esp_process_event()  local
1786 		struct scsi_cmnd *cmd = ent->cmd;  in esp_process_event()
1796 		if (ent->flags & ESP_CMD_FLAG_WRITE) {  in esp_process_event()
1813 		bytes_sent = esp_data_bytes_sent(esp, ent, cmd);  in esp_process_event()
1816 				 ent->flags, bytes_sent);  in esp_process_event()
1824 		esp_advance_dma(esp, ent, cmd, bytes_sent);  in esp_process_event()
1830 		struct esp_cmd_entry *ent = esp->active_cmd;  in esp_process_event()  local
1833 			ent->status = esp_read8(ESP_FDATA);  in esp_process_event()
1834 			ent->message = esp_read8(ESP_FDATA);  in esp_process_event()
1837 			ent->status = esp_read8(ESP_FDATA);  in esp_process_event()
1838 			ent->message = 0xff;  in esp_process_event()
1843 		if (ent->message != COMMAND_COMPLETE) {  in esp_process_event()
1846 				     ent->message);  in esp_process_event()
1856 		struct esp_cmd_entry *ent = esp->active_cmd;  in esp_process_event()  local
1857 		struct scsi_cmnd *cmd = ent->cmd;  in esp_process_event()
1859 		if (ent->message == COMMAND_COMPLETE ||  in esp_process_event()
1860 		    ent->message == DISCONNECT)  in esp_process_event()
1863 		if (ent->message == COMMAND_COMPLETE) {  in esp_process_event()
1865 					ent->status, ent->message);  in esp_process_event()
1866 			if (ent->status == SAM_STAT_TASK_SET_FULL)  in esp_process_event()
1867 				esp_event_queue_full(esp, ent);  in esp_process_event()
1869 			if (ent->status == SAM_STAT_CHECK_CONDITION &&  in esp_process_event()
1870 			    !(ent->flags & ESP_CMD_FLAG_AUTOSENSE)) {  in esp_process_event()
1871 				ent->flags |= ESP_CMD_FLAG_AUTOSENSE;  in esp_process_event()
1872 				esp_autosense(esp, ent);  in esp_process_event()
1874 				esp_cmd_is_done(esp, ent, cmd,  in esp_process_event()
1875 						compose_result(ent->status,  in esp_process_event()
1876 							       ent->message,  in esp_process_event()
1879 		} else if (ent->message == DISCONNECT) {  in esp_process_event()
1882 					   ent->tag[0], ent->tag[1]);  in esp_process_event()
1889 				     ent->message);  in esp_process_event()
2036 static void esp_reset_cleanup_one(struct esp *esp, struct esp_cmd_entry *ent)  in esp_reset_cleanup_one()  argument
2038 	struct scsi_cmnd *cmd = ent->cmd;  in esp_reset_cleanup_one()
2041 	esp_free_lun_tag(ent, cmd->device->hostdata);  in esp_reset_cleanup_one()
2044 	if (ent->flags & ESP_CMD_FLAG_AUTOSENSE)  in esp_reset_cleanup_one()
2045 		esp_unmap_sense(esp, ent);  in esp_reset_cleanup_one()
2048 	list_del(&ent->list);  in esp_reset_cleanup_one()
2049 	esp_put_ent(esp, ent);  in esp_reset_cleanup_one()
2062 	struct esp_cmd_entry *ent, *tmp;  in esp_reset_cleanup()  local
2065 	list_for_each_entry_safe(ent, tmp, &esp->queued_cmds, list) {  in esp_reset_cleanup()
2066 		struct scsi_cmnd *cmd = ent->cmd;  in esp_reset_cleanup()
2068 		list_del(&ent->list);  in esp_reset_cleanup()
2071 		esp_put_ent(esp, ent);  in esp_reset_cleanup()
2074 	list_for_each_entry_safe(ent, tmp, &esp->active_cmds, list) {  in esp_reset_cleanup()
2075 		if (ent == esp->active_cmd)  in esp_reset_cleanup()
2077 		esp_reset_cleanup_one(esp, ent);  in esp_reset_cleanup()
2500 	struct esp_cmd_entry *ent, *tmp;  in esp_eh_abort_handler()  local
2510 	ent = esp->active_cmd;  in esp_eh_abort_handler()
2511 	if (ent)  in esp_eh_abort_handler()
2514 			     ent->cmd, ent->cmd->cmnd[0]);  in esp_eh_abort_handler()
2515 	list_for_each_entry(ent, &esp->queued_cmds, list) {  in esp_eh_abort_handler()
2517 			     ent->cmd, ent->cmd->cmnd[0]);  in esp_eh_abort_handler()
2519 	list_for_each_entry(ent, &esp->active_cmds, list) {  in esp_eh_abort_handler()
2521 			     ent->cmd, ent->cmd->cmnd[0]);  in esp_eh_abort_handler()
2528 	ent = NULL;  in esp_eh_abort_handler()
2531 			ent = tmp;  in esp_eh_abort_handler()
2536 	if (ent) {  in esp_eh_abort_handler()
2540 		list_del(&ent->list);  in esp_eh_abort_handler()
2545 		esp_put_ent(esp, ent);  in esp_eh_abort_handler()
2552 	ent = esp->active_cmd;  in esp_eh_abort_handler()
2553 	if (ent && ent->cmd == cmd) {  in esp_eh_abort_handler()
2566 		ent->eh_done = &eh_done;  in esp_eh_abort_handler()
2593 		ent->eh_done = NULL;  in esp_eh_abort_handler()