99 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)  in blk_flush_policy()  argument
103 	if (blk_rq_sectors(rq))  in blk_flush_policy()
107 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_flush_policy()
110 		    (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
116 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
118 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
121 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
128 	rq->bio = rq->biotail;  in blk_flush_restore_request()
131 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
132 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
135 static bool blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
137 	if (rq->q->mq_ops) {  in blk_flush_queue_rq()
138 		blk_mq_add_to_requeue_list(rq, add_front, true);  in blk_flush_queue_rq()
142 			list_add(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
144 			list_add_tail(&rq->queuelist, &rq->q->queue_head);  in blk_flush_queue_rq()
165 static bool blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
169 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
174 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
175 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
176 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
179 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
189 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
193 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
194 		queued = blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
204 		BUG_ON(!list_empty(&rq->queuelist));  in blk_flush_complete_seq()
205 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
206 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
208 			blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
210 			__blk_end_request_all(rq, error);  in blk_flush_complete_seq()
226 	struct request *rq, *n;  in flush_end_io()  local
255 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
256 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
259 		queued |= blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
361 static void flush_data_end_io(struct request *rq, blk_status_t error)  in flush_data_end_io()  argument
363 	struct request_queue *q = rq->q;  in flush_data_end_io()
391 	elv_completed_request(q, rq);  in flush_data_end_io()
394 	rq->rq_flags &= ~RQF_STARTED;  in flush_data_end_io()
400 	if (blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error))  in flush_data_end_io()
404 static void mq_flush_data_end_io(struct request *rq, blk_status_t error)  in mq_flush_data_end_io()  argument
406 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
408 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
415 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
416 		blk_mq_put_driver_tag_hctx(hctx, rq);  in mq_flush_data_end_io()
424 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
439 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
441 	struct request_queue *q = rq->q;  in blk_insert_flush()
443 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
444 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
453 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
455 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
462 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
472 			blk_mq_end_request(rq, 0);  in blk_insert_flush()
474 			__blk_end_request(rq, 0, 0);  in blk_insert_flush()
478 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
488 			blk_mq_request_bypass_insert(rq, false);  in blk_insert_flush()
490 			list_add_tail(&rq->queuelist, &q->queue_head);  in blk_insert_flush()
498 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
499 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
500 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_insert_flush()
501 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
503 		rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
506 		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()
510 	rq->end_io = flush_data_end_io;  in blk_insert_flush()
512 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()