233 	struct mmc_queue *mq;  in power_ro_lock_store()  local
244 	mq = &md->queue;  in power_ro_lock_store()
247 	req = blk_get_request(mq->queue, REQ_OP_DRV_OUT, 0);  in power_ro_lock_store()
253 	blk_execute_rq(mq->queue, NULL, req, 0);  in power_ro_lock_store()
631 	struct mmc_queue *mq;  in mmc_blk_ioctl_cmd()  local
651 	mq = &md->queue;  in mmc_blk_ioctl_cmd()
652 	req = blk_get_request(mq->queue,  in mmc_blk_ioctl_cmd()
663 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_blk_ioctl_cmd()
681 	struct mmc_queue *mq;  in mmc_blk_ioctl_multi_cmd()  local
721 	mq = &md->queue;  in mmc_blk_ioctl_multi_cmd()
722 	req = blk_get_request(mq->queue,  in mmc_blk_ioctl_multi_cmd()
732 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_blk_ioctl_multi_cmd()
1060 static void mmc_blk_issue_drv_op(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_drv_op()  argument
1063 	struct mmc_card *card = mq->card;  in mmc_blk_issue_drv_op()
1064 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_drv_op()
1119 static void mmc_blk_issue_discard_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_discard_rq()  argument
1121 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_discard_rq()
1162 static void mmc_blk_issue_secdiscard_rq(struct mmc_queue *mq,  in mmc_blk_issue_secdiscard_rq()  argument
1165 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_secdiscard_rq()
1232 static void mmc_blk_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_issue_flush()  argument
1234 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_issue_flush()
1319 static void mmc_blk_data_prep(struct mmc_queue *mq, struct mmc_queue_req *mqrq,  in mmc_blk_data_prep()  argument
1323 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_data_prep()
1423 	brq->data.sg_len = mmc_queue_map_sg(mq, mqrq);  in mmc_blk_data_prep()
1453 static void mmc_blk_cqe_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_complete_rq()  argument
1458 	struct mmc_host *host = mq->card->host;  in mmc_blk_cqe_complete_rq()
1488 	mq->in_flight[mmc_issue_type(mq, req)] -= 1;  in mmc_blk_cqe_complete_rq()
1490 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_cqe_complete_rq()
1492 	mmc_cqe_check_busy(mq);  in mmc_blk_cqe_complete_rq()
1496 	if (!mq->cqe_busy)  in mmc_blk_cqe_complete_rq()
1500 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_cqe_complete_rq()
1503 void mmc_blk_cqe_recovery(struct mmc_queue *mq)  in mmc_blk_cqe_recovery()  argument
1505 	struct mmc_card *card = mq->card;  in mmc_blk_cqe_recovery()
1513 		mmc_blk_reset(mq->blkdata, host, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1515 		mmc_blk_reset_success(mq->blkdata, MMC_BLK_CQE_RECOVERY);  in mmc_blk_cqe_recovery()
1526 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_cqe_req_done()  local
1532 	if (mq->in_recovery)  in mmc_blk_cqe_req_done()
1533 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_cqe_req_done()
1559 static int mmc_blk_cqe_issue_flush(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_flush()  argument
1571 	return mmc_blk_cqe_start_req(mq->card->host, mrq);  in mmc_blk_cqe_issue_flush()
1574 static int mmc_blk_cqe_issue_rw_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_cqe_issue_rw_rq()  argument
1578 	mmc_blk_data_prep(mq, mqrq, 0, NULL, NULL);  in mmc_blk_cqe_issue_rw_rq()
1580 	return mmc_blk_cqe_start_req(mq->card->host, &mqrq->brq.mrq);  in mmc_blk_cqe_issue_rw_rq()
1586 			       struct mmc_queue *mq)  in mmc_blk_rw_rq_prep()  argument
1591 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_rw_rq_prep()
1594 	mmc_blk_data_prep(mq, mqrq, disable_multi, &do_rel_wr, &do_data_tag);  in mmc_blk_rw_rq_prep()
1686 static void mmc_blk_read_single(struct mmc_queue *mq, struct request *req)  in mmc_blk_read_single()  argument
1690 	struct mmc_card *card = mq->card;  in mmc_blk_read_single()
1699 		mmc_blk_rw_rq_prep(mqrq, card, 1, mq);  in mmc_blk_read_single()
1757 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_status_error()  local
1760 	if (mmc_host_is_spi(mq->card->host))  in mmc_blk_status_error()
1793 static void mmc_blk_mq_rw_recovery(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_rw_recovery()  argument
1798 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_rw_recovery()
1799 	struct mmc_card *card = mq->card;  in mmc_blk_mq_rw_recovery()
1829 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
1831 		err = mmc_blk_fix_state(mq->card, req);  in mmc_blk_mq_rw_recovery()
1846 	if (!mmc_host_is_spi(mq->card->host) &&  in mmc_blk_mq_rw_recovery()
1878 		mmc_blk_read_single(mq, req);  in mmc_blk_mq_rw_recovery()
1918 static inline void mmc_blk_rw_reset_success(struct mmc_queue *mq,  in mmc_blk_rw_reset_success()  argument
1923 	mmc_blk_reset_success(mq->blkdata, type);  in mmc_blk_rw_reset_success()
1926 static void mmc_blk_mq_complete_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_complete_rq()  argument
1941 		if (mmc_card_removed(mq->card))  in mmc_blk_mq_complete_rq()
1947 static bool mmc_blk_urgent_bkops_needed(struct mmc_queue *mq,  in mmc_blk_urgent_bkops_needed()  argument
1950 	return mmc_card_mmc(mq->card) && !mmc_host_is_spi(mq->card->host) &&  in mmc_blk_urgent_bkops_needed()
1955 static void mmc_blk_urgent_bkops(struct mmc_queue *mq,  in mmc_blk_urgent_bkops()  argument
1958 	if (mmc_blk_urgent_bkops_needed(mq, mqrq))  in mmc_blk_urgent_bkops()
1959 		mmc_start_bkops(mq->card, true);  in mmc_blk_urgent_bkops()
1964 	struct mmc_queue *mq = req->q->queuedata;  in mmc_blk_mq_complete()  local
1966 	if (mq->use_cqe)  in mmc_blk_mq_complete()
1967 		mmc_blk_cqe_complete_rq(mq, req);  in mmc_blk_mq_complete()
1969 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_complete()
1972 static void mmc_blk_mq_poll_completion(struct mmc_queue *mq,  in mmc_blk_mq_poll_completion()  argument
1976 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_poll_completion()
1979 	    mmc_blk_card_busy(mq->card, req)) {  in mmc_blk_mq_poll_completion()
1980 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_poll_completion()
1982 		mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_poll_completion()
1986 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_poll_completion()
1989 static void mmc_blk_mq_dec_in_flight(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_dec_in_flight()  argument
1997 	mq->in_flight[mmc_issue_type(mq, req)] -= 1;  in mmc_blk_mq_dec_in_flight()
1999 	put_card = (mmc_tot_in_flight(mq) == 0);  in mmc_blk_mq_dec_in_flight()
2004 		mmc_put_card(mq->card, &mq->ctx);  in mmc_blk_mq_dec_in_flight()
2007 static void mmc_blk_mq_post_req(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_post_req()  argument
2011 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_post_req()
2019 	if (mq->in_recovery)  in mmc_blk_mq_post_req()
2020 		mmc_blk_mq_complete_rq(mq, req);  in mmc_blk_mq_post_req()
2024 	mmc_blk_mq_dec_in_flight(mq, req);  in mmc_blk_mq_post_req()
2027 void mmc_blk_mq_recovery(struct mmc_queue *mq)  in mmc_blk_mq_recovery()  argument
2029 	struct request *req = mq->recovery_req;  in mmc_blk_mq_recovery()
2030 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_recovery()
2033 	mq->recovery_req = NULL;  in mmc_blk_mq_recovery()
2034 	mq->rw_wait = false;  in mmc_blk_mq_recovery()
2038 		mmc_blk_mq_rw_recovery(mq, req);  in mmc_blk_mq_recovery()
2041 	mmc_blk_urgent_bkops(mq, mqrq);  in mmc_blk_mq_recovery()
2043 	mmc_blk_mq_post_req(mq, req);  in mmc_blk_mq_recovery()
2046 static void mmc_blk_mq_complete_prev_req(struct mmc_queue *mq,  in mmc_blk_mq_complete_prev_req()  argument
2049 	if (mmc_host_done_complete(mq->card->host))  in mmc_blk_mq_complete_prev_req()
2052 	mutex_lock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2054 	if (!mq->complete_req)  in mmc_blk_mq_complete_prev_req()
2057 	mmc_blk_mq_poll_completion(mq, mq->complete_req);  in mmc_blk_mq_complete_prev_req()
2060 		*prev_req = mq->complete_req;  in mmc_blk_mq_complete_prev_req()
2062 		mmc_blk_mq_post_req(mq, mq->complete_req);  in mmc_blk_mq_complete_prev_req()
2064 	mq->complete_req = NULL;  in mmc_blk_mq_complete_prev_req()
2067 	mutex_unlock(&mq->complete_lock);  in mmc_blk_mq_complete_prev_req()
2072 	struct mmc_queue *mq = container_of(work, struct mmc_queue,  in mmc_blk_mq_complete_work()  local
2075 	mmc_blk_mq_complete_prev_req(mq, NULL);  in mmc_blk_mq_complete_work()
2084 	struct mmc_queue *mq = q->queuedata;  in mmc_blk_mq_req_done()  local
2085 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_req_done()
2098 		mq->complete_req = req;  in mmc_blk_mq_req_done()
2099 		mq->rw_wait = false;  in mmc_blk_mq_req_done()
2100 		waiting = mq->waiting;  in mmc_blk_mq_req_done()
2110 			wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2112 			kblockd_schedule_work(&mq->complete_work);  in mmc_blk_mq_req_done()
2119 	    mmc_blk_urgent_bkops_needed(mq, mqrq)) {  in mmc_blk_mq_req_done()
2121 		mq->recovery_needed = true;  in mmc_blk_mq_req_done()
2122 		mq->recovery_req = req;  in mmc_blk_mq_req_done()
2124 		wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2125 		schedule_work(&mq->recovery_work);  in mmc_blk_mq_req_done()
2129 	mmc_blk_rw_reset_success(mq, req);  in mmc_blk_mq_req_done()
2131 	mq->rw_wait = false;  in mmc_blk_mq_req_done()
2132 	wake_up(&mq->wait);  in mmc_blk_mq_req_done()
2134 	mmc_blk_mq_post_req(mq, req);  in mmc_blk_mq_req_done()
2137 static bool mmc_blk_rw_wait_cond(struct mmc_queue *mq, int *err)  in mmc_blk_rw_wait_cond()  argument
2139 	struct request_queue *q = mq->queue;  in mmc_blk_rw_wait_cond()
2148 	if (mq->recovery_needed) {  in mmc_blk_rw_wait_cond()
2152 		done = !mq->rw_wait;  in mmc_blk_rw_wait_cond()
2154 	mq->waiting = !done;  in mmc_blk_rw_wait_cond()
2160 static int mmc_blk_rw_wait(struct mmc_queue *mq, struct request **prev_req)  in mmc_blk_rw_wait()  argument
2164 	wait_event(mq->wait, mmc_blk_rw_wait_cond(mq, &err));  in mmc_blk_rw_wait()
2167 	mmc_blk_mq_complete_prev_req(mq, prev_req);  in mmc_blk_rw_wait()
2172 static int mmc_blk_mq_issue_rw_rq(struct mmc_queue *mq,  in mmc_blk_mq_issue_rw_rq()  argument
2176 	struct mmc_host *host = mq->card->host;  in mmc_blk_mq_issue_rw_rq()
2180 	mmc_blk_rw_rq_prep(mqrq, mq->card, 0, mq);  in mmc_blk_mq_issue_rw_rq()
2186 	err = mmc_blk_rw_wait(mq, &prev_req);  in mmc_blk_mq_issue_rw_rq()
2190 	mq->rw_wait = true;  in mmc_blk_mq_issue_rw_rq()
2195 		mmc_blk_mq_post_req(mq, prev_req);  in mmc_blk_mq_issue_rw_rq()
2198 		mq->rw_wait = false;  in mmc_blk_mq_issue_rw_rq()
2211 static int mmc_blk_wait_for_idle(struct mmc_queue *mq, struct mmc_host *host)  in mmc_blk_wait_for_idle()  argument
2213 	if (mq->use_cqe)  in mmc_blk_wait_for_idle()
2216 	return mmc_blk_rw_wait(mq, NULL);  in mmc_blk_wait_for_idle()
2219 enum mmc_issued mmc_blk_mq_issue_rq(struct mmc_queue *mq, struct request *req)  in mmc_blk_mq_issue_rq()  argument
2221 	struct mmc_blk_data *md = mq->blkdata;  in mmc_blk_mq_issue_rq()
2230 	switch (mmc_issue_type(mq, req)) {  in mmc_blk_mq_issue_rq()
2232 		ret = mmc_blk_wait_for_idle(mq, host);  in mmc_blk_mq_issue_rq()
2238 			mmc_blk_issue_drv_op(mq, req);  in mmc_blk_mq_issue_rq()
2241 			mmc_blk_issue_discard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2244 			mmc_blk_issue_secdiscard_rq(mq, req);  in mmc_blk_mq_issue_rq()
2247 			mmc_blk_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2258 			ret = mmc_blk_cqe_issue_flush(mq, req);  in mmc_blk_mq_issue_rq()
2262 			if (mq->use_cqe)  in mmc_blk_mq_issue_rq()
2263 				ret = mmc_blk_cqe_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
2265 				ret = mmc_blk_mq_issue_rw_rq(mq, req);  in mmc_blk_mq_issue_rq()
2757 	struct mmc_queue *mq = &md->queue;  in mmc_dbg_card_status_get()  local
2762 	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_dbg_card_status_get()
2766 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_dbg_card_status_get()
2786 	struct mmc_queue *mq = &md->queue;  in mmc_ext_csd_open()  local
2798 	req = blk_get_request(mq->queue, REQ_OP_DRV_IN, 0);  in mmc_ext_csd_open()
2805 	blk_execute_rq(mq->queue, NULL, req, 0);  in mmc_ext_csd_open()