105 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)  in blk_flush_policy()  argument
109 	if (blk_rq_sectors(rq))  in blk_flush_policy()
113 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_flush_policy()
116 		    (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
122 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
124 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
127 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
134 	rq->bio = rq->biotail;  in blk_flush_restore_request()
137 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
138 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
141 static void blk_flush_queue_rq(struct request *rq, bool add_front)  in blk_flush_queue_rq()  argument
143 	blk_mq_add_to_requeue_list(rq, add_front, true);  in blk_flush_queue_rq()
146 static void blk_account_io_flush(struct request *rq)  in blk_account_io_flush()  argument
148 	struct block_device *part = rq->q->disk->part0;  in blk_account_io_flush()
153 		      ktime_get_ns() - rq->start_time_ns);  in blk_account_io_flush()
170 static void blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
174 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
178 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
179 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
180 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
183 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
193 		list_move_tail(&rq->flush.list, pending);  in blk_flush_complete_seq()
197 		list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);  in blk_flush_complete_seq()
198 		blk_flush_queue_rq(rq, true);  in blk_flush_complete_seq()
208 		list_del_init(&rq->flush.list);  in blk_flush_complete_seq()
209 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
210 		blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
225 	struct request *rq, *n;  in flush_end_io()  local
264 	list_for_each_entry_safe(rq, n, running, flush.list) {  in flush_end_io()
265 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
268 		blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
275 bool is_flush_rq(struct request *rq)  in is_flush_rq()  argument
277 	return rq->end_io == flush_end_io;  in is_flush_rq()
358 static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq,  in mq_flush_data_end_io()  argument
361 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
362 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in mq_flush_data_end_io()
363 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
368 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
369 		blk_mq_put_driver_tag(rq);  in mq_flush_data_end_io()
377 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
393 void blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
395 	struct request_queue *q = rq->q;  in blk_insert_flush()
397 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
398 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
404 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
406 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
413 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
422 		blk_mq_end_request(rq, 0);  in blk_insert_flush()
426 	BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */  in blk_insert_flush()
435 		blk_mq_request_bypass_insert(rq, false, true);  in blk_insert_flush()
443 	memset(&rq->flush, 0, sizeof(rq->flush));  in blk_insert_flush()
444 	INIT_LIST_HEAD(&rq->flush.list);  in blk_insert_flush()
445 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_insert_flush()
446 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_insert_flush()
448 	rq->end_io = mq_flush_data_end_io;  in blk_insert_flush()
451 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()