146 static inline bool rwb_enabled(struct rq_wb *rwb)  in rwb_enabled()  argument
148 	return rwb && rwb->enable_state != WBT_STATE_OFF_DEFAULT &&  in rwb_enabled()
149 		      rwb->enable_state != WBT_STATE_OFF_MANUAL;  in rwb_enabled()
152 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)  in wb_timestamp()  argument
154 	if (rwb_enabled(rwb)) {  in wb_timestamp()
166 static bool wb_recent_wait(struct rq_wb *rwb)  in wb_recent_wait()  argument
168 	struct bdi_writeback *wb = &rwb->rqos.disk->bdi->wb;  in wb_recent_wait()
173 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,  in get_rq_wait()  argument
177 		return &rwb->rq_wait[WBT_RWQ_KSWAPD];  in get_rq_wait()
179 		return &rwb->rq_wait[WBT_RWQ_DISCARD];  in get_rq_wait()
181 	return &rwb->rq_wait[WBT_RWQ_BG];  in get_rq_wait()
184 static void rwb_wake_all(struct rq_wb *rwb)  in rwb_wake_all()  argument
189 		struct rq_wait *rqw = &rwb->rq_wait[i];  in rwb_wake_all()
196 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,  in wbt_rqw_done()  argument
209 		limit = rwb->wb_background;  in wbt_rqw_done()
210 	else if (rwb->wc && !wb_recent_wait(rwb))  in wbt_rqw_done()
213 		limit = rwb->wb_normal;  in wbt_rqw_done()
224 		if (!inflight || diff >= rwb->wb_background / 2)  in wbt_rqw_done()
231 	struct rq_wb *rwb = RQWB(rqos);  in __wbt_done()  local
237 	rqw = get_rq_wait(rwb, wb_acct);  in __wbt_done()
238 	wbt_rqw_done(rwb, rqw, wb_acct);  in __wbt_done()
247 	struct rq_wb *rwb = RQWB(rqos);  in wbt_done()  local
250 		if (rwb->sync_cookie == rq) {  in wbt_done()
251 			rwb->sync_issue = 0;  in wbt_done()
252 			rwb->sync_cookie = NULL;  in wbt_done()
256 			wb_timestamp(rwb, &rwb->last_comp);  in wbt_done()
258 		WARN_ON_ONCE(rq == rwb->sync_cookie);  in wbt_done()
276 static u64 rwb_sync_issue_lat(struct rq_wb *rwb)  in rwb_sync_issue_lat()  argument
278 	u64 now, issue = READ_ONCE(rwb->sync_issue);  in rwb_sync_issue_lat()
280 	if (!issue || !rwb->sync_cookie)  in rwb_sync_issue_lat()
287 static inline unsigned int wbt_inflight(struct rq_wb *rwb)  in wbt_inflight()  argument
292 		ret += atomic_read(&rwb->rq_wait[i].inflight);  in wbt_inflight()
304 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)  in latency_exceeded()  argument
306 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in latency_exceeded()
307 	struct rq_depth *rqd = &rwb->rq_depth;  in latency_exceeded()
319 	thislat = rwb_sync_issue_lat(rwb);  in latency_exceeded()
320 	if (thislat > rwb->cur_win_nsec ||  in latency_exceeded()
321 	    (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {  in latency_exceeded()
336 		if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||  in latency_exceeded()
337 		    wbt_inflight(rwb))  in latency_exceeded()
345 	if (stat[READ].min > rwb->min_lat_nsec) {  in latency_exceeded()
357 static void rwb_trace_step(struct rq_wb *rwb, const char *msg)  in rwb_trace_step()  argument
359 	struct backing_dev_info *bdi = rwb->rqos.disk->bdi;  in rwb_trace_step()
360 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_trace_step()
362 	trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,  in rwb_trace_step()
363 			rwb->wb_background, rwb->wb_normal, rqd->max_depth);  in rwb_trace_step()
366 static void calc_wb_limits(struct rq_wb *rwb)  in calc_wb_limits()  argument
368 	if (rwb->min_lat_nsec == 0) {  in calc_wb_limits()
369 		rwb->wb_normal = rwb->wb_background = 0;  in calc_wb_limits()
370 	} else if (rwb->rq_depth.max_depth <= 2) {  in calc_wb_limits()
371 		rwb->wb_normal = rwb->rq_depth.max_depth;  in calc_wb_limits()
372 		rwb->wb_background = 1;  in calc_wb_limits()
374 		rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;  in calc_wb_limits()
375 		rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;  in calc_wb_limits()
379 static void scale_up(struct rq_wb *rwb)  in scale_up()  argument
381 	if (!rq_depth_scale_up(&rwb->rq_depth))  in scale_up()
383 	calc_wb_limits(rwb);  in scale_up()
384 	rwb->unknown_cnt = 0;  in scale_up()
385 	rwb_wake_all(rwb);  in scale_up()
386 	rwb_trace_step(rwb, tracepoint_string("scale up"));  in scale_up()
389 static void scale_down(struct rq_wb *rwb, bool hard_throttle)  in scale_down()  argument
391 	if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))  in scale_down()
393 	calc_wb_limits(rwb);  in scale_down()
394 	rwb->unknown_cnt = 0;  in scale_down()
395 	rwb_trace_step(rwb, tracepoint_string("scale down"));  in scale_down()
398 static void rwb_arm_timer(struct rq_wb *rwb)  in rwb_arm_timer()  argument
400 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_arm_timer()
409 		rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,  in rwb_arm_timer()
416 		rwb->cur_win_nsec = rwb->win_nsec;  in rwb_arm_timer()
419 	blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);  in rwb_arm_timer()
424 	struct rq_wb *rwb = cb->data;  in wb_timer_fn()  local
425 	struct rq_depth *rqd = &rwb->rq_depth;  in wb_timer_fn()
426 	unsigned int inflight = wbt_inflight(rwb);  in wb_timer_fn()
429 	if (!rwb->rqos.disk)  in wb_timer_fn()
432 	status = latency_exceeded(rwb, cb->stat);  in wb_timer_fn()
434 	trace_wbt_timer(rwb->rqos.disk->bdi, status, rqd->scale_step, inflight);  in wb_timer_fn()
443 		scale_down(rwb, true);  in wb_timer_fn()
446 		scale_up(rwb);  in wb_timer_fn()
454 		scale_up(rwb);  in wb_timer_fn()
457 		if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)  in wb_timer_fn()
465 			scale_up(rwb);  in wb_timer_fn()
467 			scale_down(rwb, false);  in wb_timer_fn()
477 		rwb_arm_timer(rwb);  in wb_timer_fn()
480 static void wbt_update_limits(struct rq_wb *rwb)  in wbt_update_limits()  argument
482 	struct rq_depth *rqd = &rwb->rq_depth;  in wbt_update_limits()
488 	calc_wb_limits(rwb);  in wbt_update_limits()
490 	rwb_wake_all(rwb);  in wbt_update_limits()
524 static bool close_io(struct rq_wb *rwb)  in close_io()  argument
528 	return time_before(now, rwb->last_issue + HZ / 10) ||  in close_io()
529 		time_before(now, rwb->last_comp + HZ / 10);  in close_io()
534 static inline unsigned int get_limit(struct rq_wb *rwb, blk_opf_t opf)  in get_limit()  argument
539 		return rwb->wb_background;  in get_limit()
549 	if ((opf & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())  in get_limit()
550 		limit = rwb->rq_depth.max_depth;  in get_limit()
551 	else if ((opf & REQ_BACKGROUND) || close_io(rwb)) {  in get_limit()
556 		limit = rwb->wb_background;  in get_limit()
558 		limit = rwb->wb_normal;  in get_limit()
564 	struct rq_wb *rwb;  member
572 	return rq_wait_inc_below(rqw, get_limit(data->rwb, data->opf));  in wbt_inflight_cb()
578 	wbt_rqw_done(data->rwb, rqw, data->wb_acct);  in wbt_cleanup_cb()
585 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,  in __wbt_wait()  argument
588 	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);  in __wbt_wait()
590 		.rwb = rwb,  in __wbt_wait()
616 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)  in bio_to_wbt_flags()  argument
620 	if (!rwb_enabled(rwb))  in bio_to_wbt_flags()
637 	struct rq_wb *rwb = RQWB(rqos);  in wbt_cleanup()  local
638 	enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);  in wbt_cleanup()
649 	struct rq_wb *rwb = RQWB(rqos);  in wbt_wait()  local
652 	flags = bio_to_wbt_flags(rwb, bio);  in wbt_wait()
655 			wb_timestamp(rwb, &rwb->last_issue);  in wbt_wait()
659 	__wbt_wait(rwb, flags, bio->bi_opf);  in wbt_wait()
661 	if (!blk_stat_is_active(rwb->cb))  in wbt_wait()
662 		rwb_arm_timer(rwb);  in wbt_wait()
667 	struct rq_wb *rwb = RQWB(rqos);  in wbt_track()  local
668 	rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);  in wbt_track()
673 	struct rq_wb *rwb = RQWB(rqos);  in wbt_issue()  local
675 	if (!rwb_enabled(rwb))  in wbt_issue()
685 	if (wbt_is_read(rq) && !rwb->sync_issue) {  in wbt_issue()
686 		rwb->sync_cookie = rq;  in wbt_issue()
687 		rwb->sync_issue = rq->io_start_time_ns;  in wbt_issue()
693 	struct rq_wb *rwb = RQWB(rqos);  in wbt_requeue()  local
694 	if (!rwb_enabled(rwb))  in wbt_requeue()
696 	if (rq == rwb->sync_cookie) {  in wbt_requeue()
697 		rwb->sync_issue = 0;  in wbt_requeue()
698 		rwb->sync_cookie = NULL;  in wbt_requeue()
772 	struct rq_wb *rwb = RQWB(rqos);  in wbt_exit()  local
774 	blk_stat_remove_callback(rqos->disk->queue, rwb->cb);  in wbt_exit()
775 	blk_stat_free_callback(rwb->cb);  in wbt_exit()
776 	kfree(rwb);  in wbt_exit()
785 	struct rq_wb *rwb;  in wbt_disable_default()  local
788 	rwb = RQWB(rqos);  in wbt_disable_default()
789 	if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {  in wbt_disable_default()
790 		blk_stat_deactivate(rwb->cb);  in wbt_disable_default()
791 		rwb->enable_state = WBT_STATE_OFF_DEFAULT;  in wbt_disable_default()
800 	struct rq_wb *rwb = RQWB(rqos);  in wbt_curr_win_nsec_show()  local
802 	seq_printf(m, "%llu\n", rwb->cur_win_nsec);  in wbt_curr_win_nsec_show()
809 	struct rq_wb *rwb = RQWB(rqos);  in wbt_enabled_show()  local
811 	seq_printf(m, "%d\n", rwb->enable_state);  in wbt_enabled_show()
826 	struct rq_wb *rwb = RQWB(rqos);  in wbt_inflight_show()  local
831 			   atomic_read(&rwb->rq_wait[i].inflight));  in wbt_inflight_show()
838 	struct rq_wb *rwb = RQWB(rqos);  in wbt_min_lat_nsec_show()  local
840 	seq_printf(m, "%lu\n", rwb->min_lat_nsec);  in wbt_min_lat_nsec_show()
847 	struct rq_wb *rwb = RQWB(rqos);  in wbt_unknown_cnt_show()  local
849 	seq_printf(m, "%u\n", rwb->unknown_cnt);  in wbt_unknown_cnt_show()
856 	struct rq_wb *rwb = RQWB(rqos);  in wbt_normal_show()  local
858 	seq_printf(m, "%u\n", rwb->wb_normal);  in wbt_normal_show()
865 	struct rq_wb *rwb = RQWB(rqos);  in wbt_background_show()  local
867 	seq_printf(m, "%u\n", rwb->wb_background);  in wbt_background_show()
901 	struct rq_wb *rwb;  in wbt_init()  local
905 	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);  in wbt_init()
906 	if (!rwb)  in wbt_init()
909 	rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);  in wbt_init()
910 	if (!rwb->cb) {  in wbt_init()
911 		kfree(rwb);  in wbt_init()
916 		rq_wait_init(&rwb->rq_wait[i]);  in wbt_init()
918 	rwb->last_comp = rwb->last_issue = jiffies;  in wbt_init()
919 	rwb->win_nsec = RWB_WINDOW_NSEC;  in wbt_init()
920 	rwb->enable_state = WBT_STATE_ON_DEFAULT;  in wbt_init()
921 	rwb->wc = test_bit(QUEUE_FLAG_WC, &q->queue_flags);  in wbt_init()
922 	rwb->rq_depth.default_depth = RWB_DEF_DEPTH;  in wbt_init()
923 	rwb->min_lat_nsec = wbt_default_latency_nsec(q);  in wbt_init()
924 	rwb->rq_depth.queue_depth = blk_queue_depth(q);  in wbt_init()
925 	wbt_update_limits(rwb);  in wbt_init()
931 	ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);  in wbt_init()
936 	blk_stat_add_callback(q, rwb->cb);  in wbt_init()
941 	blk_stat_free_callback(rwb->cb);  in wbt_init()
942 	kfree(rwb);  in wbt_init()