300 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc);
301 static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc);
302 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc);
315 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc);
323 static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc);
324 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc);
325 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc);
755 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc)  in nv_adma_check_atapi_dma()  argument
757 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_check_atapi_dma()
854 	struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag);  in nv_host_intr()  local
867 	if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) {  in nv_host_intr()
873 	return ata_bmdma_port_intr(ap, qc);  in nv_host_intr()
1085 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc)  in nv_adma_post_internal_cmd()  argument
1087 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_post_internal_cmd()
1090 		ata_bmdma_post_internal_cmd(qc);  in nv_adma_post_internal_cmd()
1291 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc,  in nv_adma_fill_aprd()  argument
1297 	if (qc->tf.flags & ATA_TFLAG_WRITE)  in nv_adma_fill_aprd()
1299 	if (idx == qc->n_elem - 1)  in nv_adma_fill_aprd()
1310 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb)  in nv_adma_fill_sg()  argument
1312 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_fill_sg()
1317 	for_each_sg(qc->sg, sg, qc->n_elem, si) {  in nv_adma_fill_sg()
1319 			&pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)];  in nv_adma_fill_sg()
1320 		nv_adma_fill_aprd(qc, sg, si, aprd);  in nv_adma_fill_sg()
1323 		cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag)));  in nv_adma_fill_sg()
1328 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc)  in nv_adma_use_reg_mode()  argument
1330 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_use_reg_mode()
1335 	   (qc->tf.flags & ATA_TFLAG_POLLING))  in nv_adma_use_reg_mode()
1338 	if ((qc->flags & ATA_QCFLAG_DMAMAP) ||  in nv_adma_use_reg_mode()
1339 	   (qc->tf.protocol == ATA_PROT_NODATA))  in nv_adma_use_reg_mode()
1345 static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc)  in nv_adma_qc_prep()  argument
1347 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_qc_prep()
1348 	struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag];  in nv_adma_qc_prep()
1352 	if (nv_adma_use_reg_mode(qc)) {  in nv_adma_qc_prep()
1354 			(qc->flags & ATA_QCFLAG_DMAMAP));  in nv_adma_qc_prep()
1355 		nv_adma_register_mode(qc->ap);  in nv_adma_qc_prep()
1356 		ata_bmdma_qc_prep(qc);  in nv_adma_qc_prep()
1366 	cpb->tag		= qc->hw_tag;  in nv_adma_qc_prep()
1370 	if (qc->tf.protocol == ATA_PROT_NCQ)  in nv_adma_qc_prep()
1373 	nv_adma_tf_to_cpb(&qc->tf, cpb->tf);  in nv_adma_qc_prep()
1375 	if (qc->flags & ATA_QCFLAG_DMAMAP) {  in nv_adma_qc_prep()
1376 		nv_adma_fill_sg(qc, cpb);  in nv_adma_qc_prep()
1391 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc)  in nv_adma_qc_issue()  argument
1393 	struct nv_adma_port_priv *pp = qc->ap->private_data;  in nv_adma_qc_issue()
1395 	int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ);  in nv_adma_qc_issue()
1400 	if (unlikely(qc->tf.protocol == ATA_PROT_NCQ &&  in nv_adma_qc_issue()
1401 		     (qc->flags & ATA_QCFLAG_RESULT_TF))) {  in nv_adma_qc_issue()
1402 		ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n");  in nv_adma_qc_issue()
1406 	if (nv_adma_use_reg_mode(qc)) {  in nv_adma_qc_issue()
1409 			(qc->flags & ATA_QCFLAG_DMAMAP));  in nv_adma_qc_issue()
1410 		nv_adma_register_mode(qc->ap);  in nv_adma_qc_issue()
1411 		return ata_bmdma_qc_issue(qc);  in nv_adma_qc_issue()
1413 		nv_adma_mode(qc->ap);  in nv_adma_qc_issue()
1426 	writew(qc->hw_tag, mmio + NV_ADMA_APPEND);  in nv_adma_qc_issue()
1442 		struct ata_queued_cmd *qc;  in nv_generic_interrupt()  local
1444 		qc = ata_qc_from_tag(ap, ap->link.active_tag);  in nv_generic_interrupt()
1445 		if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) {  in nv_generic_interrupt()
1446 			handled += ata_bmdma_port_intr(ap, qc);  in nv_generic_interrupt()
1681 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc)  in nv_swncq_qc_to_dq()  argument
1688 	dq->defer_bits |= (1 << qc->hw_tag);  in nv_swncq_qc_to_dq()
1689 	dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag;  in nv_swncq_qc_to_dq()
1741 	struct ata_queued_cmd qc;  in __ata_bmdma_stop()  local
1743 	qc.ap = ap;  in __ata_bmdma_stop()
1744 	ata_bmdma_stop(&qc);  in __ata_bmdma_stop()
1946 static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc)  in nv_swncq_qc_prep()  argument
1948 	if (qc->tf.protocol != ATA_PROT_NCQ) {  in nv_swncq_qc_prep()
1949 		ata_bmdma_qc_prep(qc);  in nv_swncq_qc_prep()
1953 	if (!(qc->flags & ATA_QCFLAG_DMAMAP))  in nv_swncq_qc_prep()
1956 	nv_swncq_fill_sg(qc);  in nv_swncq_qc_prep()
1961 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc)  in nv_swncq_fill_sg()  argument
1963 	struct ata_port *ap = qc->ap;  in nv_swncq_fill_sg()
1969 	prd = pp->prd + ATA_MAX_PRD * qc->hw_tag;  in nv_swncq_fill_sg()
1972 	for_each_sg(qc->sg, sg, qc->n_elem, si) {  in nv_swncq_fill_sg()
1998 					  struct ata_queued_cmd *qc)  in nv_swncq_issue_atacmd()  argument
2002 	if (qc == NULL)  in nv_swncq_issue_atacmd()
2005 	writel((1 << qc->hw_tag), pp->sactive_block);  in nv_swncq_issue_atacmd()
2006 	pp->last_issue_tag = qc->hw_tag;  in nv_swncq_issue_atacmd()
2007 	pp->dhfis_bits &= ~(1 << qc->hw_tag);  in nv_swncq_issue_atacmd()
2008 	pp->dmafis_bits &= ~(1 << qc->hw_tag);  in nv_swncq_issue_atacmd()
2009 	pp->qc_active |= (0x1 << qc->hw_tag);  in nv_swncq_issue_atacmd()
2011 	trace_ata_tf_load(ap, &qc->tf);  in nv_swncq_issue_atacmd()
2012 	ap->ops->sff_tf_load(ap, &qc->tf);	 /* load tf registers */  in nv_swncq_issue_atacmd()
2013 	trace_ata_exec_command(ap, &qc->tf, qc->hw_tag);  in nv_swncq_issue_atacmd()
2014 	ap->ops->sff_exec_command(ap, &qc->tf);  in nv_swncq_issue_atacmd()
2019 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc)  in nv_swncq_qc_issue()  argument
2021 	struct ata_port *ap = qc->ap;  in nv_swncq_qc_issue()
2024 	if (qc->tf.protocol != ATA_PROT_NCQ)  in nv_swncq_qc_issue()
2025 		return ata_bmdma_qc_issue(qc);  in nv_swncq_qc_issue()
2028 		nv_swncq_issue_atacmd(ap, qc);  in nv_swncq_qc_issue()
2030 		nv_swncq_qc_to_dq(ap, qc);	/* add qc to defer queue */  in nv_swncq_qc_issue()
2062 	struct ata_queued_cmd *qc;  in nv_swncq_sdbfis()  local
2119 		qc = ata_qc_from_tag(ap, pp->last_issue_tag);  in nv_swncq_sdbfis()
2120 		nv_swncq_issue_atacmd(ap, qc);  in nv_swncq_sdbfis()
2126 		qc = nv_swncq_qc_from_dq(ap);  in nv_swncq_sdbfis()
2127 		WARN_ON(qc == NULL);  in nv_swncq_sdbfis()
2128 		nv_swncq_issue_atacmd(ap, qc);  in nv_swncq_sdbfis()
2145 	struct ata_queued_cmd *qc;  in nv_swncq_dmafis()  local
2155 	qc = ata_qc_from_tag(ap, tag);  in nv_swncq_dmafis()
2157 	if (unlikely(!qc))  in nv_swncq_dmafis()
2160 	rw = qc->tf.flags & ATA_TFLAG_WRITE;  in nv_swncq_dmafis()
2163 	iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag,  in nv_swncq_dmafis()
2178 	struct ata_queued_cmd *qc;  in nv_swncq_host_interrupt()  local
2251 				qc = nv_swncq_qc_from_dq(ap);  in nv_swncq_host_interrupt()
2252 				nv_swncq_issue_atacmd(ap, qc);  in nv_swncq_host_interrupt()