103 static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)  in blk_flush_policy()  argument
107 	if (blk_rq_sectors(rq))  in blk_flush_policy()
111 		if (rq->cmd_flags & REQ_PREFLUSH)  in blk_flush_policy()
114 		    (rq->cmd_flags & REQ_FUA))  in blk_flush_policy()
120 static unsigned int blk_flush_cur_seq(struct request *rq)  in blk_flush_cur_seq()  argument
122 	return 1 << ffz(rq->flush.seq);  in blk_flush_cur_seq()
125 static void blk_flush_restore_request(struct request *rq)  in blk_flush_restore_request()  argument
132 	rq->bio = rq->biotail;  in blk_flush_restore_request()
135 	rq->rq_flags &= ~RQF_FLUSH_SEQ;  in blk_flush_restore_request()
136 	rq->end_io = rq->flush.saved_end_io;  in blk_flush_restore_request()
139 static void blk_account_io_flush(struct request *rq)  in blk_account_io_flush()  argument
141 	struct block_device *part = rq->q->disk->part0;  in blk_account_io_flush()
146 		      ktime_get_ns() - rq->start_time_ns);  in blk_account_io_flush()
163 static void blk_flush_complete_seq(struct request *rq,  in blk_flush_complete_seq()  argument
167 	struct request_queue *q = rq->q;  in blk_flush_complete_seq()
171 	BUG_ON(rq->flush.seq & seq);  in blk_flush_complete_seq()
172 	rq->flush.seq |= seq;  in blk_flush_complete_seq()
173 	cmd_flags = rq->cmd_flags;  in blk_flush_complete_seq()
176 		seq = blk_flush_cur_seq(rq);  in blk_flush_complete_seq()
186 		list_move_tail(&rq->queuelist, pending);  in blk_flush_complete_seq()
192 		list_move(&rq->queuelist, &q->requeue_list);  in blk_flush_complete_seq()
204 		list_del_init(&rq->queuelist);  in blk_flush_complete_seq()
205 		blk_flush_restore_request(rq);  in blk_flush_complete_seq()
206 		blk_mq_end_request(rq, error);  in blk_flush_complete_seq()
221 	struct request *rq, *n;  in flush_end_io()  local
260 	list_for_each_entry_safe(rq, n, running, queuelist) {  in flush_end_io()
261 		unsigned int seq = blk_flush_cur_seq(rq);  in flush_end_io()
264 		blk_flush_complete_seq(rq, fq, seq, error);  in flush_end_io()
271 bool is_flush_rq(struct request *rq)  in is_flush_rq()  argument
273 	return rq->end_io == flush_end_io;  in is_flush_rq()
358 static enum rq_end_io_ret mq_flush_data_end_io(struct request *rq,  in mq_flush_data_end_io()  argument
361 	struct request_queue *q = rq->q;  in mq_flush_data_end_io()
362 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;  in mq_flush_data_end_io()
363 	struct blk_mq_ctx *ctx = rq->mq_ctx;  in mq_flush_data_end_io()
368 		WARN_ON(rq->tag < 0);  in mq_flush_data_end_io()
369 		blk_mq_put_driver_tag(rq);  in mq_flush_data_end_io()
382 	INIT_LIST_HEAD(&rq->queuelist);  in mq_flush_data_end_io()
383 	blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);  in mq_flush_data_end_io()
390 static void blk_rq_init_flush(struct request *rq)  in blk_rq_init_flush()  argument
392 	rq->flush.seq = 0;  in blk_rq_init_flush()
393 	rq->rq_flags |= RQF_FLUSH_SEQ;  in blk_rq_init_flush()
394 	rq->flush.saved_end_io = rq->end_io; /* Usually NULL */  in blk_rq_init_flush()
395 	rq->end_io = mq_flush_data_end_io;  in blk_rq_init_flush()
403 bool blk_insert_flush(struct request *rq)  in blk_insert_flush()  argument
405 	struct request_queue *q = rq->q;  in blk_insert_flush()
407 	unsigned int policy = blk_flush_policy(fflags, rq);  in blk_insert_flush()
408 	struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);  in blk_insert_flush()
411 	WARN_ON_ONCE(rq->bio != rq->biotail);  in blk_insert_flush()
417 	rq->cmd_flags &= ~REQ_PREFLUSH;  in blk_insert_flush()
419 		rq->cmd_flags &= ~REQ_FUA;  in blk_insert_flush()
426 	rq->cmd_flags |= REQ_SYNC;  in blk_insert_flush()
436 		blk_mq_end_request(rq, 0);  in blk_insert_flush()
450 		blk_rq_init_flush(rq);  in blk_insert_flush()
451 		rq->flush.seq |= REQ_FSEQ_PREFLUSH;  in blk_insert_flush()
461 		blk_rq_init_flush(rq);  in blk_insert_flush()
463 		blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);  in blk_insert_flush()