1534 static void ata_qc_complete_internal(struct ata_queued_cmd *qc)  in ata_qc_complete_internal()  argument
1536 	struct completion *waiting = qc->private_data;  in ata_qc_complete_internal()
1572 	struct ata_queued_cmd *qc;  in ata_exec_internal_sg()  local
1591 	qc = __ata_qc_from_tag(ap, ATA_TAG_INTERNAL);  in ata_exec_internal_sg()
1593 	qc->tag = ATA_TAG_INTERNAL;  in ata_exec_internal_sg()
1594 	qc->hw_tag = 0;  in ata_exec_internal_sg()
1595 	qc->scsicmd = NULL;  in ata_exec_internal_sg()
1596 	qc->ap = ap;  in ata_exec_internal_sg()
1597 	qc->dev = dev;  in ata_exec_internal_sg()
1598 	ata_qc_reinit(qc);  in ata_exec_internal_sg()
1610 	qc->tf = *tf;  in ata_exec_internal_sg()
1612 		memcpy(qc->cdb, cdb, ATAPI_CDB_LEN);  in ata_exec_internal_sg()
1617 		qc->tf.feature |= ATAPI_DMADIR;  in ata_exec_internal_sg()
1619 	qc->flags |= ATA_QCFLAG_RESULT_TF;  in ata_exec_internal_sg()
1620 	qc->dma_dir = dma_dir;  in ata_exec_internal_sg()
1628 		ata_sg_init(qc, sgl, n_elem);  in ata_exec_internal_sg()
1629 		qc->nbytes = buflen;  in ata_exec_internal_sg()
1632 	qc->private_data = &wait;  in ata_exec_internal_sg()
1633 	qc->complete_fn = ata_qc_complete_internal;  in ata_exec_internal_sg()
1635 	ata_qc_issue(qc);  in ata_exec_internal_sg()
1666 		if (qc->flags & ATA_QCFLAG_ACTIVE) {  in ata_exec_internal_sg()
1667 			qc->err_mask |= AC_ERR_TIMEOUT;  in ata_exec_internal_sg()
1672 				ata_qc_complete(qc);  in ata_exec_internal_sg()
1684 		ap->ops->post_internal_cmd(qc);  in ata_exec_internal_sg()
1687 	if (qc->flags & ATA_QCFLAG_FAILED) {  in ata_exec_internal_sg()
1688 		if (qc->result_tf.command & (ATA_ERR | ATA_DF))  in ata_exec_internal_sg()
1689 			qc->err_mask |= AC_ERR_DEV;  in ata_exec_internal_sg()
1691 		if (!qc->err_mask)  in ata_exec_internal_sg()
1692 			qc->err_mask |= AC_ERR_OTHER;  in ata_exec_internal_sg()
1694 		if (qc->err_mask & ~AC_ERR_OTHER)  in ata_exec_internal_sg()
1695 			qc->err_mask &= ~AC_ERR_OTHER;  in ata_exec_internal_sg()
1696 	} else if (qc->tf.command == ATA_CMD_REQ_SENSE_DATA) {  in ata_exec_internal_sg()
1697 		qc->result_tf.command |= ATA_SENSE;  in ata_exec_internal_sg()
1703 	*tf = qc->result_tf;  in ata_exec_internal_sg()
1704 	err_mask = qc->err_mask;  in ata_exec_internal_sg()
1706 	ata_qc_free(qc);  in ata_exec_internal_sg()
4948 int atapi_check_dma(struct ata_queued_cmd *qc)  in atapi_check_dma()  argument
4950 	struct ata_port *ap = qc->ap;  in atapi_check_dma()
4955 	if (!(qc->dev->horkage & ATA_HORKAGE_ATAPI_MOD16_DMA) &&  in atapi_check_dma()
4956 	    unlikely(qc->nbytes & 15))  in atapi_check_dma()
4960 		return ap->ops->check_atapi_dma(qc);  in atapi_check_dma()
4980 int ata_std_qc_defer(struct ata_queued_cmd *qc)  in ata_std_qc_defer()  argument
4982 	struct ata_link *link = qc->dev->link;  in ata_std_qc_defer()
4984 	if (ata_is_ncq(qc->tf.protocol)) {  in ata_std_qc_defer()
4995 void ata_noop_qc_prep(struct ata_queued_cmd *qc) { }  in ata_noop_qc_prep()  argument
5010 void ata_sg_init(struct ata_queued_cmd *qc, struct scatterlist *sg,  in ata_sg_init()  argument
5013 	qc->sg = sg;  in ata_sg_init()
5014 	qc->n_elem = n_elem;  in ata_sg_init()
5015 	qc->cursg = qc->sg;  in ata_sg_init()
5029 static void ata_sg_clean(struct ata_queued_cmd *qc)  in ata_sg_clean()  argument
5031 	struct ata_port *ap = qc->ap;  in ata_sg_clean()
5032 	struct scatterlist *sg = qc->sg;  in ata_sg_clean()
5033 	int dir = qc->dma_dir;  in ata_sg_clean()
5037 	VPRINTK("unmapping %u sg elements\n", qc->n_elem);  in ata_sg_clean()
5039 	if (qc->n_elem)  in ata_sg_clean()
5040 		dma_unmap_sg(ap->dev, sg, qc->orig_n_elem, dir);  in ata_sg_clean()
5042 	qc->flags &= ~ATA_QCFLAG_DMAMAP;  in ata_sg_clean()
5043 	qc->sg = NULL;  in ata_sg_clean()
5059 static int ata_sg_setup(struct ata_queued_cmd *qc)  in ata_sg_setup()  argument
5061 	struct ata_port *ap = qc->ap;  in ata_sg_setup()
5066 	n_elem = dma_map_sg(ap->dev, qc->sg, qc->n_elem, qc->dma_dir);  in ata_sg_setup()
5071 	qc->orig_n_elem = qc->n_elem;  in ata_sg_setup()
5072 	qc->n_elem = n_elem;  in ata_sg_setup()
5073 	qc->flags |= ATA_QCFLAG_DMAMAP;  in ata_sg_setup()
5080 static inline void ata_sg_clean(struct ata_queued_cmd *qc) {}  in ata_sg_clean()  argument
5081 static inline int ata_sg_setup(struct ata_queued_cmd *qc) { return -1; }  in ata_sg_setup()  argument
5119 	struct ata_queued_cmd *qc;  in ata_qc_new_init()  local
5132 	qc = __ata_qc_from_tag(ap, tag);  in ata_qc_new_init()
5133 	qc->tag = qc->hw_tag = tag;  in ata_qc_new_init()
5134 	qc->scsicmd = NULL;  in ata_qc_new_init()
5135 	qc->ap = ap;  in ata_qc_new_init()
5136 	qc->dev = dev;  in ata_qc_new_init()
5138 	ata_qc_reinit(qc);  in ata_qc_new_init()
5140 	return qc;  in ata_qc_new_init()
5153 void ata_qc_free(struct ata_queued_cmd *qc)  in ata_qc_free()  argument
5158 	WARN_ON_ONCE(qc == NULL); /* ata_qc_from_tag _might_ return NULL */  in ata_qc_free()
5159 	ap = qc->ap;  in ata_qc_free()
5161 	qc->flags = 0;  in ata_qc_free()
5162 	tag = qc->tag;  in ata_qc_free()
5164 		qc->tag = ATA_TAG_POISON;  in ata_qc_free()
5170 void __ata_qc_complete(struct ata_queued_cmd *qc)  in __ata_qc_complete()  argument
5175 	WARN_ON_ONCE(qc == NULL); /* ata_qc_from_tag _might_ return NULL */  in __ata_qc_complete()
5176 	WARN_ON_ONCE(!(qc->flags & ATA_QCFLAG_ACTIVE));  in __ata_qc_complete()
5177 	ap = qc->ap;  in __ata_qc_complete()
5178 	link = qc->dev->link;  in __ata_qc_complete()
5180 	if (likely(qc->flags & ATA_QCFLAG_DMAMAP))  in __ata_qc_complete()
5181 		ata_sg_clean(qc);  in __ata_qc_complete()
5184 	if (ata_is_ncq(qc->tf.protocol)) {  in __ata_qc_complete()
5185 		link->sactive &= ~(1 << qc->hw_tag);  in __ata_qc_complete()
5194 	if (unlikely(qc->flags & ATA_QCFLAG_CLEAR_EXCL &&  in __ata_qc_complete()
5202 	qc->flags &= ~ATA_QCFLAG_ACTIVE;  in __ata_qc_complete()
5203 	ap->qc_active &= ~(1ULL << qc->tag);  in __ata_qc_complete()
5206 	qc->complete_fn(qc);  in __ata_qc_complete()
5209 static void fill_result_tf(struct ata_queued_cmd *qc)  in fill_result_tf()  argument
5211 	struct ata_port *ap = qc->ap;  in fill_result_tf()
5213 	qc->result_tf.flags = qc->tf.flags;  in fill_result_tf()
5214 	ap->ops->qc_fill_rtf(qc);  in fill_result_tf()
5217 static void ata_verify_xfer(struct ata_queued_cmd *qc)  in ata_verify_xfer()  argument
5219 	struct ata_device *dev = qc->dev;  in ata_verify_xfer()
5221 	if (!ata_is_data(qc->tf.protocol))  in ata_verify_xfer()
5224 	if ((dev->mwdma_mask || dev->udma_mask) && ata_is_pio(qc->tf.protocol))  in ata_verify_xfer()
5245 void ata_qc_complete(struct ata_queued_cmd *qc)  in ata_qc_complete()  argument
5247 	struct ata_port *ap = qc->ap;  in ata_qc_complete()
5250 	ledtrig_disk_activity(!!(qc->tf.flags & ATA_TFLAG_WRITE));  in ata_qc_complete()
5266 		struct ata_device *dev = qc->dev;  in ata_qc_complete()
5269 		if (unlikely(qc->err_mask))  in ata_qc_complete()
5270 			qc->flags |= ATA_QCFLAG_FAILED;  in ata_qc_complete()
5276 		if (unlikely(ata_tag_internal(qc->tag))) {  in ata_qc_complete()
5277 			fill_result_tf(qc);  in ata_qc_complete()
5278 			trace_ata_qc_complete_internal(qc);  in ata_qc_complete()
5279 			__ata_qc_complete(qc);  in ata_qc_complete()
5287 		if (unlikely(qc->flags & ATA_QCFLAG_FAILED)) {  in ata_qc_complete()
5288 			fill_result_tf(qc);  in ata_qc_complete()
5289 			trace_ata_qc_complete_failed(qc);  in ata_qc_complete()
5290 			ata_qc_schedule_eh(qc);  in ata_qc_complete()
5297 		if (qc->flags & ATA_QCFLAG_RESULT_TF)  in ata_qc_complete()
5298 			fill_result_tf(qc);  in ata_qc_complete()
5300 		trace_ata_qc_complete_done(qc);  in ata_qc_complete()
5304 		switch (qc->tf.command) {  in ata_qc_complete()
5306 			if (qc->tf.feature != SETFEATURES_WC_ON &&  in ata_qc_complete()
5307 			    qc->tf.feature != SETFEATURES_WC_OFF &&  in ata_qc_complete()
5308 			    qc->tf.feature != SETFEATURES_RA_ON &&  in ata_qc_complete()
5309 			    qc->tf.feature != SETFEATURES_RA_OFF)  in ata_qc_complete()
5325 			ata_verify_xfer(qc);  in ata_qc_complete()
5327 		__ata_qc_complete(qc);  in ata_qc_complete()
5329 		if (qc->flags & ATA_QCFLAG_EH_SCHEDULED)  in ata_qc_complete()
5333 		if (qc->err_mask || qc->flags & ATA_QCFLAG_RESULT_TF)  in ata_qc_complete()
5334 			fill_result_tf(qc);  in ata_qc_complete()
5336 		__ata_qc_complete(qc);  in ata_qc_complete()
5384 		struct ata_queued_cmd *qc;  in ata_qc_complete_multiple()  local
5387 		qc = ata_qc_from_tag(ap, tag);  in ata_qc_complete_multiple()
5388 		if (qc) {  in ata_qc_complete_multiple()
5389 			ata_qc_complete(qc);  in ata_qc_complete_multiple()
5410 void ata_qc_issue(struct ata_queued_cmd *qc)  in ata_qc_issue()  argument
5412 	struct ata_port *ap = qc->ap;  in ata_qc_issue()
5413 	struct ata_link *link = qc->dev->link;  in ata_qc_issue()
5414 	u8 prot = qc->tf.protocol;  in ata_qc_issue()
5423 		WARN_ON_ONCE(link->sactive & (1 << qc->hw_tag));  in ata_qc_issue()
5427 		link->sactive |= 1 << qc->hw_tag;  in ata_qc_issue()
5432 		link->active_tag = qc->tag;  in ata_qc_issue()
5435 	qc->flags |= ATA_QCFLAG_ACTIVE;  in ata_qc_issue()
5436 	ap->qc_active |= 1ULL << qc->tag;  in ata_qc_issue()
5442 	if (ata_is_data(prot) && (!qc->sg || !qc->n_elem || !qc->nbytes))  in ata_qc_issue()
5447 		if (ata_sg_setup(qc))  in ata_qc_issue()
5451 	if (unlikely(qc->dev->flags & ATA_DFLAG_SLEEPING)) {  in ata_qc_issue()
5458 	ap->ops->qc_prep(qc);  in ata_qc_issue()
5459 	trace_ata_qc_issue(qc);  in ata_qc_issue()
5460 	qc->err_mask |= ap->ops->qc_issue(qc);  in ata_qc_issue()
5461 	if (unlikely(qc->err_mask))  in ata_qc_issue()
5466 	qc->err_mask |= AC_ERR_SYSTEM;  in ata_qc_issue()
5468 	ata_qc_complete(qc);  in ata_qc_issue()
7197 static unsigned int ata_dummy_qc_issue(struct ata_queued_cmd *qc)  in ata_dummy_qc_issue()  argument