476 unsigned int ata_sff_data_xfer(struct ata_queued_cmd *qc, unsigned char *buf,  in ata_sff_data_xfer()  argument
479 	struct ata_port *ap = qc->dev->link->ap;  in ata_sff_data_xfer()
531 unsigned int ata_sff_data_xfer32(struct ata_queued_cmd *qc, unsigned char *buf,  in ata_sff_data_xfer32()  argument
534 	struct ata_device *dev = qc->dev;  in ata_sff_data_xfer32()
541 		return ata_sff_data_xfer(qc, buf, buflen, rw);  in ata_sff_data_xfer32()
578 static void ata_pio_xfer(struct ata_queued_cmd *qc, struct page *page,  in ata_pio_xfer()  argument
581 	bool do_write = (qc->tf.flags & ATA_TFLAG_WRITE);  in ata_pio_xfer()
585 	qc->ap->ops->sff_data_xfer(qc, buf + offset, xfer_size, do_write);  in ata_pio_xfer()
601 static void ata_pio_sector(struct ata_queued_cmd *qc)  in ata_pio_sector()  argument
603 	struct ata_port *ap = qc->ap;  in ata_pio_sector()
607 	if (!qc->cursg) {  in ata_pio_sector()
608 		qc->curbytes = qc->nbytes;  in ata_pio_sector()
611 	if (qc->curbytes == qc->nbytes - qc->sect_size)  in ata_pio_sector()
614 	page = sg_page(qc->cursg);  in ata_pio_sector()
615 	offset = qc->cursg->offset + qc->cursg_ofs;  in ata_pio_sector()
621 	trace_ata_sff_pio_transfer_data(qc, offset, qc->sect_size);  in ata_pio_sector()
628 	if (offset + qc->sect_size > PAGE_SIZE) {  in ata_pio_sector()
631 		ata_pio_xfer(qc, page, offset, split_len);  in ata_pio_sector()
632 		ata_pio_xfer(qc, nth_page(page, 1), 0,  in ata_pio_sector()
633 			     qc->sect_size - split_len);  in ata_pio_sector()
635 		ata_pio_xfer(qc, page, offset, qc->sect_size);  in ata_pio_sector()
638 	qc->curbytes += qc->sect_size;  in ata_pio_sector()
639 	qc->cursg_ofs += qc->sect_size;  in ata_pio_sector()
641 	if (qc->cursg_ofs == qc->cursg->length) {  in ata_pio_sector()
642 		qc->cursg = sg_next(qc->cursg);  in ata_pio_sector()
643 		if (!qc->cursg)  in ata_pio_sector()
645 		qc->cursg_ofs = 0;  in ata_pio_sector()
659 static void ata_pio_sectors(struct ata_queued_cmd *qc)  in ata_pio_sectors()  argument
661 	if (is_multi_taskfile(&qc->tf)) {  in ata_pio_sectors()
665 		WARN_ON_ONCE(qc->dev->multi_count == 0);  in ata_pio_sectors()
667 		nsect = min((qc->nbytes - qc->curbytes) / qc->sect_size,  in ata_pio_sectors()
668 			    qc->dev->multi_count);  in ata_pio_sectors()
670 			ata_pio_sector(qc);  in ata_pio_sectors()
672 		ata_pio_sector(qc);  in ata_pio_sectors()
674 	ata_sff_sync(qc->ap); /* flush */  in ata_pio_sectors()
688 static void atapi_send_cdb(struct ata_port *ap, struct ata_queued_cmd *qc)  in atapi_send_cdb()  argument
691 	trace_atapi_send_cdb(qc, 0, qc->dev->cdb_len);  in atapi_send_cdb()
692 	WARN_ON_ONCE(qc->dev->cdb_len < 12);  in atapi_send_cdb()
694 	ap->ops->sff_data_xfer(qc, qc->cdb, qc->dev->cdb_len, 1);  in atapi_send_cdb()
698 	switch (qc->tf.protocol) {  in atapi_send_cdb()
709 		trace_ata_bmdma_start(ap, &qc->tf, qc->tag);  in atapi_send_cdb()
710 		ap->ops->bmdma_start(qc);  in atapi_send_cdb()
729 static int __atapi_pio_bytes(struct ata_queued_cmd *qc, unsigned int bytes)  in __atapi_pio_bytes()  argument
731 	int rw = (qc->tf.flags & ATA_TFLAG_WRITE) ? WRITE : READ;  in __atapi_pio_bytes()
732 	struct ata_port *ap = qc->ap;  in __atapi_pio_bytes()
733 	struct ata_device *dev = qc->dev;  in __atapi_pio_bytes()
741 	sg = qc->cursg;  in __atapi_pio_bytes()
745 				  qc->nbytes, qc->curbytes, bytes);  in __atapi_pio_bytes()
750 	offset = sg->offset + qc->cursg_ofs;  in __atapi_pio_bytes()
757 	count = min(sg->length - qc->cursg_ofs, bytes);  in __atapi_pio_bytes()
762 	trace_atapi_pio_transfer_data(qc, offset, count);  in __atapi_pio_bytes()
766 	consumed = ap->ops->sff_data_xfer(qc, buf + offset, count, rw);  in __atapi_pio_bytes()
770 	qc->curbytes += count;  in __atapi_pio_bytes()
771 	qc->cursg_ofs += count;  in __atapi_pio_bytes()
773 	if (qc->cursg_ofs == sg->length) {  in __atapi_pio_bytes()
774 		qc->cursg = sg_next(qc->cursg);  in __atapi_pio_bytes()
775 		qc->cursg_ofs = 0;  in __atapi_pio_bytes()
798 static void atapi_pio_bytes(struct ata_queued_cmd *qc)  in atapi_pio_bytes()  argument
800 	struct ata_port *ap = qc->ap;  in atapi_pio_bytes()
801 	struct ata_device *dev = qc->dev;  in atapi_pio_bytes()
804 	int i_write, do_write = (qc->tf.flags & ATA_TFLAG_WRITE) ? 1 : 0;  in atapi_pio_bytes()
812 	ap->ops->sff_tf_read(ap, &qc->result_tf);  in atapi_pio_bytes()
813 	ireason = qc->result_tf.nsect;  in atapi_pio_bytes()
814 	bc_lo = qc->result_tf.lbam;  in atapi_pio_bytes()
815 	bc_hi = qc->result_tf.lbah;  in atapi_pio_bytes()
830 	if (unlikely(__atapi_pio_bytes(qc, bytes)))  in atapi_pio_bytes()
840 	qc->err_mask |= AC_ERR_HSM;  in atapi_pio_bytes()
853 						struct ata_queued_cmd *qc)  in ata_hsm_ok_in_wq()  argument
855 	if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_hsm_ok_in_wq()
859 		if (qc->tf.protocol == ATA_PROT_PIO &&  in ata_hsm_ok_in_wq()
860 		   (qc->tf.flags & ATA_TFLAG_WRITE))  in ata_hsm_ok_in_wq()
863 		if (ata_is_atapi(qc->tf.protocol) &&  in ata_hsm_ok_in_wq()
864 		   !(qc->dev->flags & ATA_DFLAG_CDB_INTR))  in ata_hsm_ok_in_wq()
882 static void ata_hsm_qc_complete(struct ata_queued_cmd *qc, int in_wq)  in ata_hsm_qc_complete()  argument
884 	struct ata_port *ap = qc->ap;  in ata_hsm_qc_complete()
888 		qc = ata_qc_from_tag(ap, qc->tag);  in ata_hsm_qc_complete()
889 		if (qc) {  in ata_hsm_qc_complete()
890 			if (likely(!(qc->err_mask & AC_ERR_HSM))) {  in ata_hsm_qc_complete()
892 				ata_qc_complete(qc);  in ata_hsm_qc_complete()
897 		if (likely(!(qc->err_mask & AC_ERR_HSM)))  in ata_hsm_qc_complete()
898 			ata_qc_complete(qc);  in ata_hsm_qc_complete()
914 int ata_sff_hsm_move(struct ata_port *ap, struct ata_queued_cmd *qc,  in ata_sff_hsm_move()  argument
917 	struct ata_link *link = qc->dev->link;  in ata_sff_hsm_move()
923 	WARN_ON_ONCE((qc->flags & ATA_QCFLAG_ACTIVE) == 0);  in ata_sff_hsm_move()
929 	WARN_ON_ONCE(in_wq != ata_hsm_ok_in_wq(ap, qc));  in ata_sff_hsm_move()
932 	trace_ata_sff_hsm_state(qc, status);  in ata_sff_hsm_move()
942 		poll_next = (qc->tf.flags & ATA_TFLAG_POLLING);  in ata_sff_hsm_move()
949 				qc->err_mask |= AC_ERR_DEV;  in ata_sff_hsm_move()
954 				qc->err_mask |= AC_ERR_HSM;  in ata_sff_hsm_move()
973 			if (!(qc->dev->horkage & ATA_HORKAGE_STUCK_ERR)) {  in ata_sff_hsm_move()
977 				qc->err_mask |= AC_ERR_HSM;  in ata_sff_hsm_move()
983 		if (qc->tf.protocol == ATA_PROT_PIO) {  in ata_sff_hsm_move()
993 			ata_pio_sectors(qc);  in ata_sff_hsm_move()
996 			atapi_send_cdb(ap, qc);  in ata_sff_hsm_move()
1005 		if (qc->tf.protocol == ATAPI_PROT_PIO) {  in ata_sff_hsm_move()
1025 				qc->err_mask |= AC_ERR_HSM;  in ata_sff_hsm_move()
1030 			atapi_pio_bytes(qc);  in ata_sff_hsm_move()
1042 					qc->err_mask |= AC_ERR_DEV;  in ata_sff_hsm_move()
1048 					if (qc->dev->horkage &  in ata_sff_hsm_move()
1050 						qc->err_mask |=  in ata_sff_hsm_move()
1060 					qc->err_mask |= AC_ERR_HSM |  in ata_sff_hsm_move()
1080 				qc->err_mask |= AC_ERR_DEV;  in ata_sff_hsm_move()
1082 				if (!(qc->tf.flags & ATA_TFLAG_WRITE)) {  in ata_sff_hsm_move()
1083 					ata_pio_sectors(qc);  in ata_sff_hsm_move()
1091 					qc->err_mask |= AC_ERR_HSM;  in ata_sff_hsm_move()
1102 					qc->err_mask |= AC_ERR_NODEV_HINT;  in ata_sff_hsm_move()
1112 			ata_pio_sectors(qc);  in ata_sff_hsm_move()
1115 			    (!(qc->tf.flags & ATA_TFLAG_WRITE))) {  in ata_sff_hsm_move()
1127 			qc->err_mask |= __ac_err_mask(status);  in ata_sff_hsm_move()
1133 		trace_ata_sff_hsm_command_complete(qc, status);  in ata_sff_hsm_move()
1135 		WARN_ON_ONCE(qc->err_mask & (AC_ERR_DEV | AC_ERR_HSM));  in ata_sff_hsm_move()
1140 		ata_hsm_qc_complete(qc, in_wq);  in ata_sff_hsm_move()
1149 		ata_hsm_qc_complete(qc, in_wq);  in ata_sff_hsm_move()
1214 	struct ata_queued_cmd *qc;  in ata_sff_pio_task()  local
1222 	qc = ata_qc_from_tag(ap, link->active_tag);  in ata_sff_pio_task()
1223 	if (!qc) {  in ata_sff_pio_task()
1257 	poll_next = ata_sff_hsm_move(ap, qc, status, 1);  in ata_sff_pio_task()
1281 unsigned int ata_sff_qc_issue(struct ata_queued_cmd *qc)  in ata_sff_qc_issue()  argument
1283 	struct ata_port *ap = qc->ap;  in ata_sff_qc_issue()
1284 	struct ata_link *link = qc->dev->link;  in ata_sff_qc_issue()
1290 		qc->tf.flags |= ATA_TFLAG_POLLING;  in ata_sff_qc_issue()
1293 	ata_dev_select(ap, qc->dev->devno, 1, 0);  in ata_sff_qc_issue()
1296 	switch (qc->tf.protocol) {  in ata_sff_qc_issue()
1298 		if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_qc_issue()
1299 			ata_qc_set_polling(qc);  in ata_sff_qc_issue()
1301 		ata_tf_to_host(ap, &qc->tf, qc->tag);  in ata_sff_qc_issue()
1304 		if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_qc_issue()
1310 		if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_qc_issue()
1311 			ata_qc_set_polling(qc);  in ata_sff_qc_issue()
1313 		ata_tf_to_host(ap, &qc->tf, qc->tag);  in ata_sff_qc_issue()
1315 		if (qc->tf.flags & ATA_TFLAG_WRITE) {  in ata_sff_qc_issue()
1327 			if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_qc_issue()
1340 		if (qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_qc_issue()
1341 			ata_qc_set_polling(qc);  in ata_sff_qc_issue()
1343 		ata_tf_to_host(ap, &qc->tf, qc->tag);  in ata_sff_qc_issue()
1348 		if ((!(qc->dev->flags & ATA_DFLAG_CDB_INTR)) ||  in ata_sff_qc_issue()
1349 		    (qc->tf.flags & ATA_TFLAG_POLLING))  in ata_sff_qc_issue()
1371 void ata_sff_qc_fill_rtf(struct ata_queued_cmd *qc)  in ata_sff_qc_fill_rtf()  argument
1373 	qc->ap->ops->sff_tf_read(qc->ap, &qc->result_tf);  in ata_sff_qc_fill_rtf()
1394 					struct ata_queued_cmd *qc,  in __ata_sff_port_intr()  argument
1399 	trace_ata_sff_port_intr(qc, hsmv_on_idle);  in __ata_sff_port_intr()
1412 		if (!(qc->dev->flags & ATA_DFLAG_CDB_INTR))  in __ata_sff_port_intr()
1426 			qc->err_mask |= AC_ERR_HSM;  in __ata_sff_port_intr()
1436 	ata_sff_hsm_move(ap, qc, status, 0);  in __ata_sff_port_intr()
1454 unsigned int ata_sff_port_intr(struct ata_port *ap, struct ata_queued_cmd *qc)  in ata_sff_port_intr()  argument
1456 	return __ata_sff_port_intr(ap, qc, false);  in ata_sff_port_intr()
1476 		struct ata_queued_cmd *qc;  in __ata_sff_interrupt()  local
1478 		qc = ata_qc_from_tag(ap, ap->link.active_tag);  in __ata_sff_interrupt()
1479 		if (qc) {  in __ata_sff_interrupt()
1480 			if (!(qc->tf.flags & ATA_TFLAG_POLLING))  in __ata_sff_interrupt()
1481 				handled |= port_intr(ap, qc);  in __ata_sff_interrupt()
1568 	struct ata_queued_cmd *qc;  in ata_sff_lost_interrupt()  local
1571 	qc = ata_qc_from_tag(ap, ap->link.active_tag);  in ata_sff_lost_interrupt()
1573 	if (!qc || qc->tf.flags & ATA_TFLAG_POLLING)  in ata_sff_lost_interrupt()
1587 	ata_sff_port_intr(ap, qc);  in ata_sff_lost_interrupt()
2021 void ata_sff_drain_fifo(struct ata_queued_cmd *qc)  in ata_sff_drain_fifo()  argument
2027 	if (qc == NULL || qc->dma_dir == DMA_TO_DEVICE)  in ata_sff_drain_fifo()
2030 	ap = qc->ap;  in ata_sff_drain_fifo()
2058 	struct ata_queued_cmd *qc;  in ata_sff_error_handler()  local
2061 	qc = __ata_qc_from_tag(ap, ap->link.active_tag);  in ata_sff_error_handler()
2062 	if (qc && !(qc->flags & ATA_QCFLAG_EH))  in ata_sff_error_handler()
2063 		qc = NULL;  in ata_sff_error_handler()
2075 		ap->ops->sff_drain_fifo(qc);  in ata_sff_error_handler()
2497 static void ata_bmdma_fill_sg(struct ata_queued_cmd *qc)  in ata_bmdma_fill_sg()  argument
2499 	struct ata_port *ap = qc->ap;  in ata_bmdma_fill_sg()
2505 	for_each_sg(qc->sg, sg, qc->n_elem, si) {  in ata_bmdma_fill_sg()
2547 static void ata_bmdma_fill_sg_dumb(struct ata_queued_cmd *qc)  in ata_bmdma_fill_sg_dumb()  argument
2549 	struct ata_port *ap = qc->ap;  in ata_bmdma_fill_sg_dumb()
2555 	for_each_sg(qc->sg, sg, qc->n_elem, si) {  in ata_bmdma_fill_sg_dumb()
2602 enum ata_completion_errors ata_bmdma_qc_prep(struct ata_queued_cmd *qc)  in ata_bmdma_qc_prep()  argument
2604 	if (!(qc->flags & ATA_QCFLAG_DMAMAP))  in ata_bmdma_qc_prep()
2607 	ata_bmdma_fill_sg(qc);  in ata_bmdma_qc_prep()
2622 enum ata_completion_errors ata_bmdma_dumb_qc_prep(struct ata_queued_cmd *qc)  in ata_bmdma_dumb_qc_prep()  argument
2624 	if (!(qc->flags & ATA_QCFLAG_DMAMAP))  in ata_bmdma_dumb_qc_prep()
2627 	ata_bmdma_fill_sg_dumb(qc);  in ata_bmdma_dumb_qc_prep()
2647 unsigned int ata_bmdma_qc_issue(struct ata_queued_cmd *qc)  in ata_bmdma_qc_issue()  argument
2649 	struct ata_port *ap = qc->ap;  in ata_bmdma_qc_issue()
2650 	struct ata_link *link = qc->dev->link;  in ata_bmdma_qc_issue()
2653 	if (!ata_is_dma(qc->tf.protocol))  in ata_bmdma_qc_issue()
2654 		return ata_sff_qc_issue(qc);  in ata_bmdma_qc_issue()
2657 	ata_dev_select(ap, qc->dev->devno, 1, 0);  in ata_bmdma_qc_issue()
2660 	switch (qc->tf.protocol) {  in ata_bmdma_qc_issue()
2662 		WARN_ON_ONCE(qc->tf.flags & ATA_TFLAG_POLLING);  in ata_bmdma_qc_issue()
2664 		trace_ata_tf_load(ap, &qc->tf);  in ata_bmdma_qc_issue()
2665 		ap->ops->sff_tf_load(ap, &qc->tf);  /* load tf registers */  in ata_bmdma_qc_issue()
2666 		trace_ata_bmdma_setup(ap, &qc->tf, qc->tag);  in ata_bmdma_qc_issue()
2667 		ap->ops->bmdma_setup(qc);	    /* set up bmdma */  in ata_bmdma_qc_issue()
2668 		trace_ata_bmdma_start(ap, &qc->tf, qc->tag);  in ata_bmdma_qc_issue()
2669 		ap->ops->bmdma_start(qc);	    /* initiate bmdma */  in ata_bmdma_qc_issue()
2674 		WARN_ON_ONCE(qc->tf.flags & ATA_TFLAG_POLLING);  in ata_bmdma_qc_issue()
2676 		trace_ata_tf_load(ap, &qc->tf);  in ata_bmdma_qc_issue()
2677 		ap->ops->sff_tf_load(ap, &qc->tf);  /* load tf registers */  in ata_bmdma_qc_issue()
2678 		trace_ata_bmdma_setup(ap, &qc->tf, qc->tag);  in ata_bmdma_qc_issue()
2679 		ap->ops->bmdma_setup(qc);	    /* set up bmdma */  in ata_bmdma_qc_issue()
2683 		if (!(qc->dev->flags & ATA_DFLAG_CDB_INTR))  in ata_bmdma_qc_issue()
2709 unsigned int ata_bmdma_port_intr(struct ata_port *ap, struct ata_queued_cmd *qc)  in ata_bmdma_port_intr()  argument
2716 	if (ap->hsm_task_state == HSM_ST_LAST && ata_is_dma(qc->tf.protocol)) {  in ata_bmdma_port_intr()
2726 		trace_ata_bmdma_stop(ap, &qc->tf, qc->tag);  in ata_bmdma_port_intr()
2727 		ap->ops->bmdma_stop(qc);  in ata_bmdma_port_intr()
2732 			qc->err_mask |= AC_ERR_HOST_BUS;  in ata_bmdma_port_intr()
2737 	handled = __ata_sff_port_intr(ap, qc, bmdma_stopped);  in ata_bmdma_port_intr()
2739 	if (unlikely(qc->err_mask) && ata_is_dma(qc->tf.protocol))  in ata_bmdma_port_intr()
2780 	struct ata_queued_cmd *qc;  in ata_bmdma_error_handler()  local
2784 	qc = __ata_qc_from_tag(ap, ap->link.active_tag);  in ata_bmdma_error_handler()
2785 	if (qc && !(qc->flags & ATA_QCFLAG_EH))  in ata_bmdma_error_handler()
2786 		qc = NULL;  in ata_bmdma_error_handler()
2791 	if (qc && ata_is_dma(qc->tf.protocol)) {  in ata_bmdma_error_handler()
2802 		if (qc->err_mask == AC_ERR_TIMEOUT && (host_stat & ATA_DMA_ERR)) {  in ata_bmdma_error_handler()
2803 			qc->err_mask = AC_ERR_HOST_BUS;  in ata_bmdma_error_handler()
2807 		trace_ata_bmdma_stop(ap, &qc->tf, qc->tag);  in ata_bmdma_error_handler()
2808 		ap->ops->bmdma_stop(qc);  in ata_bmdma_error_handler()
2834 void ata_bmdma_post_internal_cmd(struct ata_queued_cmd *qc)  in ata_bmdma_post_internal_cmd()  argument
2836 	struct ata_port *ap = qc->ap;  in ata_bmdma_post_internal_cmd()
2839 	if (ata_is_dma(qc->tf.protocol)) {  in ata_bmdma_post_internal_cmd()
2841 		trace_ata_bmdma_stop(ap, &qc->tf, qc->tag);  in ata_bmdma_post_internal_cmd()
2842 		ap->ops->bmdma_stop(qc);  in ata_bmdma_post_internal_cmd()
2877 void ata_bmdma_setup(struct ata_queued_cmd *qc)  in ata_bmdma_setup()  argument
2879 	struct ata_port *ap = qc->ap;  in ata_bmdma_setup()
2880 	unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE);  in ata_bmdma_setup()
2895 	ap->ops->sff_exec_command(ap, &qc->tf);  in ata_bmdma_setup()
2906 void ata_bmdma_start(struct ata_queued_cmd *qc)  in ata_bmdma_start()  argument
2908 	struct ata_port *ap = qc->ap;  in ata_bmdma_start()
2943 void ata_bmdma_stop(struct ata_queued_cmd *qc)  in ata_bmdma_stop()  argument
2945 	struct ata_port *ap = qc->ap;  in ata_bmdma_stop()