146 static void blk_clear_congested(struct request_list *rl, int sync)  in blk_clear_congested()  argument
149 	clear_wb_congested(rl->blkg->wb_congested, sync);  in blk_clear_congested()
155 	if (rl == &rl->q->root_rl)  in blk_clear_congested()
156 		clear_wb_congested(rl->q->backing_dev_info->wb.congested, sync);  in blk_clear_congested()
160 static void blk_set_congested(struct request_list *rl, int sync)  in blk_set_congested()  argument
163 	set_wb_congested(rl->blkg->wb_congested, sync);  in blk_set_congested()
166 	if (rl == &rl->q->root_rl)  in blk_set_congested()
167 		set_wb_congested(rl->q->backing_dev_info->wb.congested, sync);  in blk_set_congested()
618 		struct request_list *rl;  in __blk_drain_queue()  local
620 		blk_queue_for_each_rl(rl, q)  in __blk_drain_queue()
621 			for (i = 0; i < ARRAY_SIZE(rl->wait); i++)  in __blk_drain_queue()
622 				wake_up_all(&rl->wait[i]);  in __blk_drain_queue()
701 		struct request_list *rl;  in blk_set_queue_dying()  local
704 		blk_queue_for_each_rl(rl, q) {  in blk_set_queue_dying()
705 			if (rl->rq_pool) {  in blk_set_queue_dying()
706 				wake_up_all(&rl->wait[BLK_RW_SYNC]);  in blk_set_queue_dying()
707 				wake_up_all(&rl->wait[BLK_RW_ASYNC]);  in blk_set_queue_dying()
868 int blk_init_rl(struct request_list *rl, struct request_queue *q,  in blk_init_rl()  argument
871 	if (unlikely(rl->rq_pool) || q->mq_ops)  in blk_init_rl()
874 	rl->q = q;  in blk_init_rl()
875 	rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;  in blk_init_rl()
876 	rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;  in blk_init_rl()
877 	init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);  in blk_init_rl()
878 	init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);  in blk_init_rl()
881 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,  in blk_init_rl()
885 		rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ,  in blk_init_rl()
889 	if (!rl->rq_pool)  in blk_init_rl()
892 	if (rl != &q->root_rl)  in blk_init_rl()
898 void blk_exit_rl(struct request_queue *q, struct request_list *rl)  in blk_exit_rl()  argument
900 	if (rl->rq_pool) {  in blk_exit_rl()
901 		mempool_destroy(rl->rq_pool);  in blk_exit_rl()
902 		if (rl != &q->root_rl)  in blk_exit_rl()
1208 static inline void blk_free_request(struct request_list *rl, struct request *rq)  in blk_free_request()  argument
1211 		elv_put_request(rl->q, rq);  in blk_free_request()
1216 	mempool_free(rq, rl->rq_pool);  in blk_free_request()
1253 static void __freed_request(struct request_list *rl, int sync)  in __freed_request()  argument
1255 	struct request_queue *q = rl->q;  in __freed_request()
1257 	if (rl->count[sync] < queue_congestion_off_threshold(q))  in __freed_request()
1258 		blk_clear_congested(rl, sync);  in __freed_request()
1260 	if (rl->count[sync] + 1 <= q->nr_requests) {  in __freed_request()
1261 		if (waitqueue_active(&rl->wait[sync]))  in __freed_request()
1262 			wake_up(&rl->wait[sync]);  in __freed_request()
1264 		blk_clear_rl_full(rl, sync);  in __freed_request()
1272 static void freed_request(struct request_list *rl, bool sync,  in freed_request()  argument
1275 	struct request_queue *q = rl->q;  in freed_request()
1278 	rl->count[sync]--;  in freed_request()
1282 	__freed_request(rl, sync);  in freed_request()
1284 	if (unlikely(rl->starved[sync ^ 1]))  in freed_request()
1285 		__freed_request(rl, sync ^ 1);  in freed_request()
1290 	struct request_list *rl;  in blk_update_nr_requests()  local
1301 	blk_queue_for_each_rl(rl, q) {  in blk_update_nr_requests()
1302 		if (rl->count[BLK_RW_SYNC] >= on_thresh)  in blk_update_nr_requests()
1303 			blk_set_congested(rl, BLK_RW_SYNC);  in blk_update_nr_requests()
1304 		else if (rl->count[BLK_RW_SYNC] < off_thresh)  in blk_update_nr_requests()
1305 			blk_clear_congested(rl, BLK_RW_SYNC);  in blk_update_nr_requests()
1307 		if (rl->count[BLK_RW_ASYNC] >= on_thresh)  in blk_update_nr_requests()
1308 			blk_set_congested(rl, BLK_RW_ASYNC);  in blk_update_nr_requests()
1309 		else if (rl->count[BLK_RW_ASYNC] < off_thresh)  in blk_update_nr_requests()
1310 			blk_clear_congested(rl, BLK_RW_ASYNC);  in blk_update_nr_requests()
1312 		if (rl->count[BLK_RW_SYNC] >= q->nr_requests) {  in blk_update_nr_requests()
1313 			blk_set_rl_full(rl, BLK_RW_SYNC);  in blk_update_nr_requests()
1315 			blk_clear_rl_full(rl, BLK_RW_SYNC);  in blk_update_nr_requests()
1316 			wake_up(&rl->wait[BLK_RW_SYNC]);  in blk_update_nr_requests()
1319 		if (rl->count[BLK_RW_ASYNC] >= q->nr_requests) {  in blk_update_nr_requests()
1320 			blk_set_rl_full(rl, BLK_RW_ASYNC);  in blk_update_nr_requests()
1322 			blk_clear_rl_full(rl, BLK_RW_ASYNC);  in blk_update_nr_requests()
1323 			wake_up(&rl->wait[BLK_RW_ASYNC]);  in blk_update_nr_requests()
1346 static struct request *__get_request(struct request_list *rl, unsigned int op,  in __get_request()  argument
1349 	struct request_queue *q = rl->q;  in __get_request()
1367 	if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {  in __get_request()
1368 		if (rl->count[is_sync]+1 >= q->nr_requests) {  in __get_request()
1375 			if (!blk_rl_full(rl, is_sync)) {  in __get_request()
1377 				blk_set_rl_full(rl, is_sync);  in __get_request()
1390 		blk_set_congested(rl, is_sync);  in __get_request()
1398 	if (rl->count[is_sync] >= (3 * q->nr_requests / 2))  in __get_request()
1402 	rl->count[is_sync]++;  in __get_request()
1403 	rl->starved[is_sync] = 0;  in __get_request()
1430 	rq = mempool_alloc(rl->rq_pool, gfp_mask);  in __get_request()
1435 	blk_rq_set_rl(rq, rl);  in __get_request()
1498 	freed_request(rl, is_sync, rq_flags);  in __get_request()
1508 	if (unlikely(rl->count[is_sync] == 0))  in __get_request()
1509 		rl->starved[is_sync] = 1;  in __get_request()
1533 	struct request_list *rl;  in get_request()  local
1539 	rl = blk_get_rl(q, bio);	/* transferred to @rq on success */  in get_request()
1541 	rq = __get_request(rl, op, bio, flags, gfp);  in get_request()
1546 		blk_put_rl(rl);  in get_request()
1551 		blk_put_rl(rl);  in get_request()
1556 	prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,  in get_request()
1572 	finish_wait(&rl->wait[is_sync], &wait);  in get_request()
1768 		struct request_list *rl = blk_rq_rl(req);  in __blk_put_request()  local
1774 		blk_free_request(rl, req);  in __blk_put_request()
1775 		freed_request(rl, sync, rq_flags);  in __blk_put_request()
1776 		blk_put_rl(rl);  in __blk_put_request()