26 static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq)  in mmc_cqe_dcmd_busy()  argument
29 	return mq->in_flight[MMC_ISSUE_DCMD];  in mmc_cqe_dcmd_busy()
32 void mmc_cqe_check_busy(struct mmc_queue *mq)  in mmc_cqe_check_busy()  argument
34 	if ((mq->cqe_busy & MMC_CQE_DCMD_BUSY) && !mmc_cqe_dcmd_busy(mq))  in mmc_cqe_check_busy()
35 		mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;  in mmc_cqe_check_busy()
37 	mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;  in mmc_cqe_check_busy()
61 enum mmc_issue_type mmc_issue_type(struct mmc_queue *mq, struct request *req)  in mmc_issue_type()  argument
63 	struct mmc_host *host = mq->card->host;  in mmc_issue_type()
65 	if (mq->use_cqe)  in mmc_issue_type()
74 static void __mmc_cqe_recovery_notifier(struct mmc_queue *mq)  in __mmc_cqe_recovery_notifier()  argument
76 	if (!mq->recovery_needed) {  in __mmc_cqe_recovery_notifier()
77 		mq->recovery_needed = true;  in __mmc_cqe_recovery_notifier()
78 		schedule_work(&mq->recovery_work);  in __mmc_cqe_recovery_notifier()
88 	struct mmc_queue *mq = q->queuedata;  in mmc_cqe_recovery_notifier()  local
91 	spin_lock_irqsave(&mq->lock, flags);  in mmc_cqe_recovery_notifier()
92 	__mmc_cqe_recovery_notifier(mq);  in mmc_cqe_recovery_notifier()
93 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_cqe_recovery_notifier()
100 	struct mmc_queue *mq = req->q->queuedata;  in mmc_cqe_timed_out()  local
101 	struct mmc_host *host = mq->card->host;  in mmc_cqe_timed_out()
102 	enum mmc_issue_type issue_type = mmc_issue_type(mq, req);  in mmc_cqe_timed_out()
110 				__mmc_cqe_recovery_notifier(mq);  in mmc_cqe_timed_out()
126 	struct mmc_queue *mq = q->queuedata;  in mmc_mq_timed_out()  local
130 	spin_lock_irqsave(&mq->lock, flags);  in mmc_mq_timed_out()
132 	if (mq->recovery_needed || !mq->use_cqe)  in mmc_mq_timed_out()
137 	spin_unlock_irqrestore(&mq->lock, flags);  in mmc_mq_timed_out()
144 	struct mmc_queue *mq = container_of(work, struct mmc_queue,  in mmc_mq_recovery_handler()  local
146 	struct request_queue *q = mq->queue;  in mmc_mq_recovery_handler()
148 	mmc_get_card(mq->card, &mq->ctx);  in mmc_mq_recovery_handler()
150 	mq->in_recovery = true;  in mmc_mq_recovery_handler()
152 	if (mq->use_cqe)  in mmc_mq_recovery_handler()
153 		mmc_blk_cqe_recovery(mq);  in mmc_mq_recovery_handler()
155 		mmc_blk_mq_recovery(mq);  in mmc_mq_recovery_handler()
157 	mq->in_recovery = false;  in mmc_mq_recovery_handler()
159 	spin_lock_irq(&mq->lock);  in mmc_mq_recovery_handler()
160 	mq->recovery_needed = false;  in mmc_mq_recovery_handler()
161 	spin_unlock_irq(&mq->lock);  in mmc_mq_recovery_handler()
163 	mmc_put_card(mq->card, &mq->ctx);  in mmc_mq_recovery_handler()
210 static int __mmc_init_request(struct mmc_queue *mq, struct request *req,  in __mmc_init_request()  argument
214 	struct mmc_card *card = mq->card;  in __mmc_init_request()
241 	struct mmc_queue *mq = set->driver_data;  in mmc_mq_exit_request()  local
243 	mmc_exit_request(mq->queue, req);  in mmc_mq_exit_request()
251 	struct mmc_queue *mq = q->queuedata;  in mmc_mq_queue_rq()  local
252 	struct mmc_card *card = mq->card;  in mmc_mq_queue_rq()
259 	if (mmc_card_removed(mq->card)) {  in mmc_mq_queue_rq()
264 	issue_type = mmc_issue_type(mq, req);  in mmc_mq_queue_rq()
266 	spin_lock_irq(&mq->lock);  in mmc_mq_queue_rq()
268 	if (mq->recovery_needed || mq->busy) {  in mmc_mq_queue_rq()
269 		spin_unlock_irq(&mq->lock);  in mmc_mq_queue_rq()
275 		if (mmc_cqe_dcmd_busy(mq)) {  in mmc_mq_queue_rq()
276 			mq->cqe_busy |= MMC_CQE_DCMD_BUSY;  in mmc_mq_queue_rq()
277 			spin_unlock_irq(&mq->lock);  in mmc_mq_queue_rq()
296 	mq->busy = true;  in mmc_mq_queue_rq()
298 	mq->in_flight[issue_type] += 1;  in mmc_mq_queue_rq()
299 	get_card = (mmc_tot_in_flight(mq) == 1);  in mmc_mq_queue_rq()
300 	cqe_retune_ok = (mmc_cqe_qcnt(mq) == 1);  in mmc_mq_queue_rq()
302 	spin_unlock_irq(&mq->lock);  in mmc_mq_queue_rq()
310 		mmc_get_card(card, &mq->ctx);  in mmc_mq_queue_rq()
312 	if (mq->use_cqe) {  in mmc_mq_queue_rq()
319 	issued = mmc_blk_mq_issue_rq(mq, req);  in mmc_mq_queue_rq()
336 		spin_lock_irq(&mq->lock);  in mmc_mq_queue_rq()
337 		mq->in_flight[issue_type] -= 1;  in mmc_mq_queue_rq()
338 		if (mmc_tot_in_flight(mq) == 0)  in mmc_mq_queue_rq()
340 		mq->busy = false;  in mmc_mq_queue_rq()
341 		spin_unlock_irq(&mq->lock);  in mmc_mq_queue_rq()
343 			mmc_put_card(card, &mq->ctx);  in mmc_mq_queue_rq()
345 		WRITE_ONCE(mq->busy, false);  in mmc_mq_queue_rq()
359 static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)  in mmc_setup_queue()  argument
364 	blk_queue_flag_set(QUEUE_FLAG_NONROT, mq->queue);  in mmc_setup_queue()
365 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, mq->queue);  in mmc_setup_queue()
367 		mmc_queue_setup_discard(mq->queue, card);  in mmc_setup_queue()
370 		blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_HIGH);  in mmc_setup_queue()
371 	blk_queue_max_hw_sectors(mq->queue,  in mmc_setup_queue()
374 		WARN(!blk_queue_can_use_dma_map_merging(mq->queue,  in mmc_setup_queue()
377 	blk_queue_max_segments(mq->queue, mmc_get_max_segments(host));  in mmc_setup_queue()
382 	blk_queue_logical_block_size(mq->queue, block_size);  in mmc_setup_queue()
389 		blk_queue_max_segment_size(mq->queue,  in mmc_setup_queue()
392 	dma_set_max_seg_size(mmc_dev(host), queue_max_segment_size(mq->queue));  in mmc_setup_queue()
394 	INIT_WORK(&mq->recovery_work, mmc_mq_recovery_handler);  in mmc_setup_queue()
395 	INIT_WORK(&mq->complete_work, mmc_blk_mq_complete_work);  in mmc_setup_queue()
397 	mutex_init(&mq->complete_lock);  in mmc_setup_queue()
399 	init_waitqueue_head(&mq->wait);  in mmc_setup_queue()
417 int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card)  in mmc_init_queue()  argument
422 	mq->card = card;  in mmc_init_queue()
423 	mq->use_cqe = host->cqe_enabled;  in mmc_init_queue()
425 	spin_lock_init(&mq->lock);  in mmc_init_queue()
427 	memset(&mq->tag_set, 0, sizeof(mq->tag_set));  in mmc_init_queue()
428 	mq->tag_set.ops = &mmc_mq_ops;  in mmc_init_queue()
433 	if (mq->use_cqe)  in mmc_init_queue()
434 		mq->tag_set.queue_depth =  in mmc_init_queue()
437 		mq->tag_set.queue_depth = MMC_QUEUE_DEPTH;  in mmc_init_queue()
438 	mq->tag_set.numa_node = NUMA_NO_NODE;  in mmc_init_queue()
439 	mq->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;  in mmc_init_queue()
440 	mq->tag_set.nr_hw_queues = 1;  in mmc_init_queue()
441 	mq->tag_set.cmd_size = sizeof(struct mmc_queue_req);  in mmc_init_queue()
442 	mq->tag_set.driver_data = mq;  in mmc_init_queue()
456 	ret = blk_mq_alloc_tag_set(&mq->tag_set);  in mmc_init_queue()
460 	mq->queue = blk_mq_init_queue(&mq->tag_set);  in mmc_init_queue()
461 	if (IS_ERR(mq->queue)) {  in mmc_init_queue()
462 		ret = PTR_ERR(mq->queue);  in mmc_init_queue()
467 		mq->queue->backing_dev_info->capabilities |=  in mmc_init_queue()
470 	mq->queue->queuedata = mq;  in mmc_init_queue()
471 	blk_queue_rq_timeout(mq->queue, 60 * HZ);  in mmc_init_queue()
473 	mmc_setup_queue(mq, card);  in mmc_init_queue()
477 	blk_mq_free_tag_set(&mq->tag_set);  in mmc_init_queue()
481 void mmc_queue_suspend(struct mmc_queue *mq)  in mmc_queue_suspend()  argument
483 	blk_mq_quiesce_queue(mq->queue);  in mmc_queue_suspend()
489 	mmc_claim_host(mq->card->host);  in mmc_queue_suspend()
490 	mmc_release_host(mq->card->host);  in mmc_queue_suspend()
493 void mmc_queue_resume(struct mmc_queue *mq)  in mmc_queue_resume()  argument
495 	blk_mq_unquiesce_queue(mq->queue);  in mmc_queue_resume()
498 void mmc_cleanup_queue(struct mmc_queue *mq)  in mmc_cleanup_queue()  argument
500 	struct request_queue *q = mq->queue;  in mmc_cleanup_queue()
510 	blk_mq_free_tag_set(&mq->tag_set);  in mmc_cleanup_queue()
517 	flush_work(&mq->complete_work);  in mmc_cleanup_queue()
519 	mq->card = NULL;  in mmc_cleanup_queue()
525 unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)  in mmc_queue_map_sg()  argument
529 	return blk_rq_map_sg(mq->queue, req, mqrq->sg);  in mmc_queue_map_sg()