173 			       struct mmc_queue *mq);
236 	struct mmc_queue *mq;  in power_ro_lock_store()  local
247 	mq = &md->queue;  in power_ro_lock_store()
250 	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, 0);  in power_ro_lock_store()
256 	blk_execute_rq(mq->queue, NULL, req, 0);  in power_ro_lock_store()
600 	struct mmc_queue *mq;  in mmc_blk_ioctl_cmd()  local
620 	mq = &md->queue;  in mmc_blk_ioctl_cmd()
621 	req = blk_get_request(mq->queue,  in mmc_blk_ioctl_cmd()
632 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_blk_ioctl_cmd()
650 	struct mmc_queue *mq;  in mmc_blk_ioctl_multi_cmd()  local
690 	mq = &md->queue;  in mmc_blk_ioctl_multi_cmd()
691 	req = blk_get_request(mq->queue,  in mmc_blk_ioctl_multi_cmd()
701 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_blk_ioctl_multi_cmd()
977 static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_drv_op()  argument
980 	struct mmc_card *card = mq->card;  in mmc_blk_issue_drv_op()
981 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_drv_op()
1036 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_discard_rq()  argument
1038 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_discard_rq()
1073 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,  in mmc_blk_issue_secdiscard_rq()  argument
1076 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_secdiscard_rq()
1143 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_flush()  argument
1145 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_flush()
1230 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,  in mmc_blk_data_prep()  argument
1234 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_data_prep()
1334 	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);  in mmc_blk_data_prep()
1364 static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_complete_rq()  argument
1369 	struct mmc_host *host = mq->card->host;  in mmc_blk_cqe_complete_rq()
1370 	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);  in mmc_blk_cqe_complete_rq()
1398 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_cqe_complete_rq()
1400 	mq->in_flight[issue_type] -= 1;  in mmc_blk_cqe_complete_rq()
1402 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_cqe_complete_rq()
1404 	mmc_cqe_check_busy(mq);  in mmc_blk_cqe_complete_rq()
1406 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_cqe_complete_rq()
1408 	if (!mq->cqe_busy)  in mmc_blk_cqe_complete_rq()
1412 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_cqe_complete_rq()
1415 void mmc_blk_cqe_recovery(struct mmc_queue *mq)  in mmc_blk_cqe_recovery()  argument
1417 	struct mmc_card *card = mq->card;  in mmc_blk_cqe_recovery()
1425 		mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1427 		mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1438 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_cqe_req_done()  local
1444 	if (mq->in_recovery)  in mmc_blk_cqe_req_done()
1445 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_cqe_req_done()
1471 static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_flush()  argument
1483 	return mmc_blk_cqe_start_req(mq->card->host, mrq);  in mmc_blk_cqe_issue_flush()
1486 static int mmc_blk_hsq_issue_rw_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_hsq_issue_rw_rq()  argument
1489 	struct mmc_host *host = mq->card->host;  in mmc_blk_hsq_issue_rw_rq()
1492 	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);  in mmc_blk_hsq_issue_rw_rq()
1503 static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_rw_rq()  argument
1506 	struct mmc_host *host = mq->card->host;  in mmc_blk_cqe_issue_rw_rq()
1509 		return mmc_blk_hsq_issue_rw_rq(mq, req);  in mmc_blk_cqe_issue_rw_rq()
1511 	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);  in mmc_blk_cqe_issue_rw_rq()
1513 	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);  in mmc_blk_cqe_issue_rw_rq()
1519 			       struct mmc_queue *mq)  in mmc_blk_rw_rq_prep()  argument
1524 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_rw_rq_prep()
1527 	mmc_blk_data_prep(mq, mqrq, disable_multi, &do_rel_wr, &do_data_tag);  in mmc_blk_rw_rq_prep()
1619 static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)  in mmc_blk_read_single()  argument
1623 	struct mmc_card *card = mq->card;  in mmc_blk_read_single()
1632 		mmc_blk_rw_rq_prep(mqrq, card, 1, mq);  in mmc_blk_read_single()
1690 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_status_error()  local
1693 	if (mmc_host_is_spi(mq->card->host))  in mmc_blk_status_error()
1726 static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_rw_recovery()  argument
1731 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_rw_recovery()
1732 	struct mmc_card *card = mq->card;  in mmc_blk_mq_rw_recovery()
1762 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
1764 		err = mmc_blk_fix_state(mq->card, req);  in mmc_blk_mq_rw_recovery()
1779 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
1811 		mmc_blk_read_single(mq, req);  in mmc_blk_mq_rw_recovery()
1851 static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,  in mmc_blk_rw_reset_success()  argument
1856 	mmc_blk_reset_success(mq->blkdata, type);  in mmc_blk_rw_reset_success()
1859 static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_complete_rq()  argument
1874 		if (mmc_card_removed(mq->card))  in mmc_blk_mq_complete_rq()
1880 static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,  in mmc_blk_urgent_bkops_needed()  argument
1883 	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&  in mmc_blk_urgent_bkops_needed()
1888 static void mmc_blk_urgent_bkops(struct mmc_queue *mq,  in mmc_blk_urgent_bkops()  argument
1891 	if (mmc_blk_urgent_bkops_needed(mq, mqrq))  in mmc_blk_urgent_bkops()
1892 		mmc_run_bkops(mq->card);  in mmc_blk_urgent_bkops()
1901 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_hsq_req_done()  local
1902 	struct mmc_host *host = mq->card->host;  in mmc_blk_hsq_req_done()
1906 	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {  in mmc_blk_hsq_req_done()
1907 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_hsq_req_done()
1908 		mq->recovery_needed = true;  in mmc_blk_hsq_req_done()
1909 		mq->recovery_req = req;  in mmc_blk_hsq_req_done()
1910 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_hsq_req_done()
1914 		schedule_work(&mq->recovery_work);  in mmc_blk_hsq_req_done()
1918 	mmc_blk_rw_reset_success(mq, req);  in mmc_blk_hsq_req_done()
1924 	if (mq->in_recovery)  in mmc_blk_hsq_req_done()
1925 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_hsq_req_done()
1932 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_mq_complete()  local
1934 	if (mq->use_cqe)  in mmc_blk_mq_complete()
1935 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_mq_complete()
1937 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_complete()
1940 static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,  in mmc_blk_mq_poll_completion()  argument
1944 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_poll_completion()
1947 	    mmc_blk_card_busy(mq->card, req)) {  in mmc_blk_mq_poll_completion()
1948 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_poll_completion()
1950 		mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_poll_completion()
1954 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_poll_completion()
1957 static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_dec_in_flight()  argument
1962 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_dec_in_flight()
1964 	mq->in_flight[mmc_issue_type(mq, req)] -= 1;  in mmc_blk_mq_dec_in_flight()
1966 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_mq_dec_in_flight()
1968 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_dec_in_flight()
1971 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_mq_dec_in_flight()
1974 static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_post_req()  argument
1978 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_post_req()
1986 	if (mq->in_recovery)  in mmc_blk_mq_post_req()
1987 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_post_req()
1991 	mmc_blk_mq_dec_in_flight(mq, req);  in mmc_blk_mq_post_req()
1994 void mmc_blk_mq_recovery(struct mmc_queue *mq)  in mmc_blk_mq_recovery()  argument
1996 	struct request *req = mq->recovery_req;  in mmc_blk_mq_recovery()
1997 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_recovery()
2000 	mq->recovery_req = NULL;  in mmc_blk_mq_recovery()
2001 	mq->rw_wait = false;  in mmc_blk_mq_recovery()
2005 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_recovery()
2008 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_recovery()
2010 	mmc_blk_mq_post_req(mq, req);  in mmc_blk_mq_recovery()
2013 static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,  in mmc_blk_mq_complete_prev_req()  argument
2016 	if (mmc_host_done_complete(mq->card->host))  in mmc_blk_mq_complete_prev_req()
2019 	mutex_lock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2021 	if (!mq->complete_req)  in mmc_blk_mq_complete_prev_req()
2024 	mmc_blk_mq_poll_completion(mq, mq->complete_req);  in mmc_blk_mq_complete_prev_req()
2027 		*prev_req = mq->complete_req;  in mmc_blk_mq_complete_prev_req()
2029 		mmc_blk_mq_post_req(mq, mq->complete_req);  in mmc_blk_mq_complete_prev_req()
2031 	mq->complete_req = NULL;  in mmc_blk_mq_complete_prev_req()
2034 	mutex_unlock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2039 	struct mmc_queue *mq = container_of(work, struct mmc_queue,  in mmc_blk_mq_complete_work()  local
2042 	mmc_blk_mq_complete_prev_req(mq, NULL);  in mmc_blk_mq_complete_work()
2051 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_mq_req_done()  local
2052 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_req_done()
2064 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_req_done()
2065 		mq->complete_req = req;  in mmc_blk_mq_req_done()
2066 		mq->rw_wait = false;  in mmc_blk_mq_req_done()
2067 		waiting = mq->waiting;  in mmc_blk_mq_req_done()
2068 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_req_done()
2077 			wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2079 			queue_work(mq->card->complete_wq, &mq->complete_work);  in mmc_blk_mq_req_done()
2086 	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {  in mmc_blk_mq_req_done()
2087 		spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_mq_req_done()
2088 		mq->recovery_needed = true;  in mmc_blk_mq_req_done()
2089 		mq->recovery_req = req;  in mmc_blk_mq_req_done()
2090 		spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_mq_req_done()
2091 		wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2092 		schedule_work(&mq->recovery_work);  in mmc_blk_mq_req_done()
2096 	mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_req_done()
2098 	mq->rw_wait = false;  in mmc_blk_mq_req_done()
2099 	wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2101 	mmc_blk_mq_post_req(mq, req);  in mmc_blk_mq_req_done()
2104 static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)  in mmc_blk_rw_wait_cond()  argument
2113 	spin_lock_irqsave(&mq->lock, flags);  in mmc_blk_rw_wait_cond()
2114 	if (mq->recovery_needed) {  in mmc_blk_rw_wait_cond()
2118 		done = !mq->rw_wait;  in mmc_blk_rw_wait_cond()
2120 	mq->waiting = !done;  in mmc_blk_rw_wait_cond()
2121 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_blk_rw_wait_cond()
2126 static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)  in mmc_blk_rw_wait()  argument
2130 	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));  in mmc_blk_rw_wait()
2133 	mmc_blk_mq_complete_prev_req(mq, prev_req);  in mmc_blk_rw_wait()
2138 static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,  in mmc_blk_mq_issue_rw_rq()  argument
2142 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_issue_rw_rq()
2146 	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);  in mmc_blk_mq_issue_rw_rq()
2152 	err = mmc_blk_rw_wait(mq, &prev_req);  in mmc_blk_mq_issue_rw_rq()
2156 	mq->rw_wait = true;  in mmc_blk_mq_issue_rw_rq()
2161 		mmc_blk_mq_post_req(mq, prev_req);  in mmc_blk_mq_issue_rw_rq()
2164 		mq->rw_wait = false;  in mmc_blk_mq_issue_rw_rq()
2177 static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)  in mmc_blk_wait_for_idle()  argument
2179 	if (mq->use_cqe)  in mmc_blk_wait_for_idle()
2182 	return mmc_blk_rw_wait(mq, NULL);  in mmc_blk_wait_for_idle()
2185 enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_issue_rq()  argument
2187 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_issue_rq()
2196 	switch (mmc_issue_type(mq, req)) {  in mmc_blk_mq_issue_rq()
2198 		ret = mmc_blk_wait_for_idle(mq, host);  in mmc_blk_mq_issue_rq()
2204 			mmc_blk_issue_drv_op(mq, req);  in mmc_blk_mq_issue_rq()
2207 			mmc_blk_issue_discard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2210 			mmc_blk_issue_secdiscard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2213 			mmc_blk_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2224 			ret = mmc_blk_cqe_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2228 			if (mq->use_cqe)  in mmc_blk_mq_issue_rq()
2229 				ret = mmc_blk_cqe_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
2231 				ret = mmc_blk_mq_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
2716 	struct mmc_queue *mq = &md->queue;  in mmc_dbg_card_status_get()  local
2721 	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_dbg_card_status_get()
2725 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_dbg_card_status_get()
2745 	struct mmc_queue *mq = &md->queue;  in mmc_ext_csd_open()  local
2757 	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_ext_csd_open()
2764 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_ext_csd_open()