98 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)  in blk_flush_policy()  argument
102 	if (blk_rq_sectors(rq))  in blk_flush_policy()
106 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_flush_policy()
109 		    (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
115 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
117 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
120 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
127 	rq->bio = rq->biotail;  in blk_flush_restore_request()
130 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
131 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
134 static void blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
136 	blk_mq_add_to_requeue_list(rq, add_front, true);  in blk_flush_queue_rq()
155 static void blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
159 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
163 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
164 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
165 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
168 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
178 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
182 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
183 		blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
193 		BUG_ON(!list_empty(&rq->queuelist));  in blk_flush_complete_seq()
194 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
195 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
196 		blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
210 	struct request *rq, *n;  in flush_end_io()  local
243 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
244 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
247 		blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
326 static void mq_flush_data_end_io(struct request *rq, blk_status_t error)  in mq_flush_data_end_io()  argument
328 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
329 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in mq_flush_data_end_io()
330 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
335 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
336 		blk_mq_put_driver_tag(rq);  in mq_flush_data_end_io()
344 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
359 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
361 	struct request_queue *q = rq->q;  in blk_insert_flush()
363 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
364 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
370 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
372 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
379 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
388 		blk_mq_end_request(rq, 0);  in blk_insert_flush()
392 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
401 		blk_mq_request_bypass_insert(rq, false);  in blk_insert_flush()
409 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
410 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
411 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_insert_flush()
412 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
414 	rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
417 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()