77 static inline bool rwb_enabled(struct rq_wb *rwb)  in rwb_enabled()  argument
79 	return rwb && rwb->wb_normal != 0;  in rwb_enabled()
82 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)  in wb_timestamp()  argument
84 	if (rwb_enabled(rwb)) {  in wb_timestamp()
96 static bool wb_recent_wait(struct rq_wb *rwb)  in wb_recent_wait()  argument
98 	struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb;  in wb_recent_wait()
103 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,  in get_rq_wait()  argument
107 		return &rwb->rq_wait[WBT_RWQ_KSWAPD];  in get_rq_wait()
109 		return &rwb->rq_wait[WBT_RWQ_DISCARD];  in get_rq_wait()
111 	return &rwb->rq_wait[WBT_RWQ_BG];  in get_rq_wait()
114 static void rwb_wake_all(struct rq_wb *rwb)  in rwb_wake_all()  argument
119 		struct rq_wait *rqw = &rwb->rq_wait[i];  in rwb_wake_all()
126 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,  in wbt_rqw_done()  argument
137 	if (unlikely(!rwb_enabled(rwb))) {  in wbt_rqw_done()
138 		rwb_wake_all(rwb);  in wbt_rqw_done()
148 		limit = rwb->wb_background;  in wbt_rqw_done()
149 	else if (rwb->wc && !wb_recent_wait(rwb))  in wbt_rqw_done()
152 		limit = rwb->wb_normal;  in wbt_rqw_done()
163 		if (!inflight || diff >= rwb->wb_background / 2)  in wbt_rqw_done()
170 	struct rq_wb *rwb = RQWB(rqos);  in __wbt_done()  local
176 	rqw = get_rq_wait(rwb, wb_acct);  in __wbt_done()
177 	wbt_rqw_done(rwb, rqw, wb_acct);  in __wbt_done()
186 	struct rq_wb *rwb = RQWB(rqos);  in wbt_done()  local
189 		if (rwb->sync_cookie == rq) {  in wbt_done()
190 			rwb->sync_issue = 0;  in wbt_done()
191 			rwb->sync_cookie = NULL;  in wbt_done()
195 			wb_timestamp(rwb, &rwb->last_comp);  in wbt_done()
197 		WARN_ON_ONCE(rq == rwb->sync_cookie);  in wbt_done()
215 static u64 rwb_sync_issue_lat(struct rq_wb *rwb)  in rwb_sync_issue_lat()  argument
217 	u64 now, issue = READ_ONCE(rwb->sync_issue);  in rwb_sync_issue_lat()
219 	if (!issue || !rwb->sync_cookie)  in rwb_sync_issue_lat()
233 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)  in latency_exceeded()  argument
235 	struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;  in latency_exceeded()
236 	struct rq_depth *rqd = &rwb->rq_depth;  in latency_exceeded()
248 	thislat = rwb_sync_issue_lat(rwb);  in latency_exceeded()
249 	if (thislat > rwb->cur_win_nsec ||  in latency_exceeded()
250 	    (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {  in latency_exceeded()
265 		if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||  in latency_exceeded()
266 		    wbt_inflight(rwb))  in latency_exceeded()
274 	if (stat[READ].min > rwb->min_lat_nsec) {  in latency_exceeded()
286 static void rwb_trace_step(struct rq_wb *rwb, const char *msg)  in rwb_trace_step()  argument
288 	struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;  in rwb_trace_step()
289 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_trace_step()
291 	trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,  in rwb_trace_step()
292 			rwb->wb_background, rwb->wb_normal, rqd->max_depth);  in rwb_trace_step()
295 static void calc_wb_limits(struct rq_wb *rwb)  in calc_wb_limits()  argument
297 	if (rwb->min_lat_nsec == 0) {  in calc_wb_limits()
298 		rwb->wb_normal = rwb->wb_background = 0;  in calc_wb_limits()
299 	} else if (rwb->rq_depth.max_depth <= 2) {  in calc_wb_limits()
300 		rwb->wb_normal = rwb->rq_depth.max_depth;  in calc_wb_limits()
301 		rwb->wb_background = 1;  in calc_wb_limits()
303 		rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;  in calc_wb_limits()
304 		rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;  in calc_wb_limits()
308 static void scale_up(struct rq_wb *rwb)  in scale_up()  argument
310 	rq_depth_scale_up(&rwb->rq_depth);  in scale_up()
311 	calc_wb_limits(rwb);  in scale_up()
312 	rwb->unknown_cnt = 0;  in scale_up()
313 	rwb_wake_all(rwb);  in scale_up()
314 	rwb_trace_step(rwb, "scale up");  in scale_up()
317 static void scale_down(struct rq_wb *rwb, bool hard_throttle)  in scale_down()  argument
319 	rq_depth_scale_down(&rwb->rq_depth, hard_throttle);  in scale_down()
320 	calc_wb_limits(rwb);  in scale_down()
321 	rwb->unknown_cnt = 0;  in scale_down()
322 	rwb_trace_step(rwb, "scale down");  in scale_down()
325 static void rwb_arm_timer(struct rq_wb *rwb)  in rwb_arm_timer()  argument
327 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_arm_timer()
336 		rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,  in rwb_arm_timer()
343 		rwb->cur_win_nsec = rwb->win_nsec;  in rwb_arm_timer()
346 	blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);  in rwb_arm_timer()
351 	struct rq_wb *rwb = cb->data;  in wb_timer_fn()  local
352 	struct rq_depth *rqd = &rwb->rq_depth;  in wb_timer_fn()
353 	unsigned int inflight = wbt_inflight(rwb);  in wb_timer_fn()
356 	status = latency_exceeded(rwb, cb->stat);  in wb_timer_fn()
358 	trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step,  in wb_timer_fn()
368 		scale_down(rwb, true);  in wb_timer_fn()
371 		scale_up(rwb);  in wb_timer_fn()
379 		scale_up(rwb);  in wb_timer_fn()
382 		if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)  in wb_timer_fn()
390 			scale_up(rwb);  in wb_timer_fn()
392 			scale_down(rwb, false);  in wb_timer_fn()
402 		rwb_arm_timer(rwb);  in wb_timer_fn()
405 static void __wbt_update_limits(struct rq_wb *rwb)  in __wbt_update_limits()  argument
407 	struct rq_depth *rqd = &rwb->rq_depth;  in __wbt_update_limits()
413 	calc_wb_limits(rwb);  in __wbt_update_limits()
415 	rwb_wake_all(rwb);  in __wbt_update_limits()
445 static bool close_io(struct rq_wb *rwb)  in close_io()  argument
449 	return time_before(now, rwb->last_issue + HZ / 10) ||  in close_io()
450 		time_before(now, rwb->last_comp + HZ / 10);  in close_io()
455 static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)  in get_limit()  argument
463 	if (!rwb_enabled(rwb))  in get_limit()
467 		return rwb->wb_background;  in get_limit()
477 	if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())  in get_limit()
478 		limit = rwb->rq_depth.max_depth;  in get_limit()
479 	else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {  in get_limit()
484 		limit = rwb->wb_background;  in get_limit()
486 		limit = rwb->wb_normal;  in get_limit()
494 	struct rq_wb *rwb;  member
510 	if (!rq_wait_inc_below(data->rqw, get_limit(data->rwb, data->rw)))  in wbt_wake_function()
523 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,  in __wbt_wait()  argument
528 	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);  in __wbt_wait()
535 		.rwb = rwb,  in __wbt_wait()
542 	if (!has_sleeper && rq_wait_inc_below(rqw, get_limit(rwb, rw)))  in __wbt_wait()
551 		    rq_wait_inc_below(rqw, get_limit(rwb, rw))) {  in __wbt_wait()
560 				wbt_rqw_done(rwb, rqw, wb_acct);  in __wbt_wait()
577 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)  in wbt_should_throttle()  argument
595 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)  in bio_to_wbt_flags()  argument
599 	if (!rwb_enabled(rwb))  in bio_to_wbt_flags()
604 	} else if (wbt_should_throttle(rwb, bio)) {  in bio_to_wbt_flags()
616 	struct rq_wb *rwb = RQWB(rqos);  in wbt_cleanup()  local
617 	enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);  in wbt_cleanup()
629 	struct rq_wb *rwb = RQWB(rqos);  in wbt_wait()  local
632 	flags = bio_to_wbt_flags(rwb, bio);  in wbt_wait()
635 			wb_timestamp(rwb, &rwb->last_issue);  in wbt_wait()
639 	__wbt_wait(rwb, flags, bio->bi_opf, lock);  in wbt_wait()
641 	if (!blk_stat_is_active(rwb->cb))  in wbt_wait()
642 		rwb_arm_timer(rwb);  in wbt_wait()
647 	struct rq_wb *rwb = RQWB(rqos);  in wbt_track()  local
648 	rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);  in wbt_track()
653 	struct rq_wb *rwb = RQWB(rqos);  in wbt_issue()  local
655 	if (!rwb_enabled(rwb))  in wbt_issue()
665 	if (wbt_is_read(rq) && !rwb->sync_issue) {  in wbt_issue()
666 		rwb->sync_cookie = rq;  in wbt_issue()
667 		rwb->sync_issue = rq->io_start_time_ns;  in wbt_issue()
673 	struct rq_wb *rwb = RQWB(rqos);  in wbt_requeue()  local
674 	if (!rwb_enabled(rwb))  in wbt_requeue()
676 	if (rq == rwb->sync_cookie) {  in wbt_requeue()
677 		rwb->sync_issue = 0;  in wbt_requeue()
678 		rwb->sync_cookie = NULL;  in wbt_requeue()
745 	struct rq_wb *rwb = RQWB(rqos);  in wbt_exit()  local
748 	blk_stat_remove_callback(q, rwb->cb);  in wbt_exit()
749 	blk_stat_free_callback(rwb->cb);  in wbt_exit()
750 	kfree(rwb);  in wbt_exit()
759 	struct rq_wb *rwb;  in wbt_disable_default()  local
762 	rwb = RQWB(rqos);  in wbt_disable_default()
763 	if (rwb->enable_state == WBT_STATE_ON_DEFAULT)  in wbt_disable_default()
764 		rwb->wb_normal = 0;  in wbt_disable_default()
781 	struct rq_wb *rwb;  in wbt_init()  local
784 	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);  in wbt_init()
785 	if (!rwb)  in wbt_init()
788 	rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);  in wbt_init()
789 	if (!rwb->cb) {  in wbt_init()
790 		kfree(rwb);  in wbt_init()
795 		rq_wait_init(&rwb->rq_wait[i]);  in wbt_init()
797 	rwb->rqos.id = RQ_QOS_WBT;  in wbt_init()
798 	rwb->rqos.ops = &wbt_rqos_ops;  in wbt_init()
799 	rwb->rqos.q = q;  in wbt_init()
800 	rwb->last_comp = rwb->last_issue = jiffies;  in wbt_init()
801 	rwb->win_nsec = RWB_WINDOW_NSEC;  in wbt_init()
802 	rwb->enable_state = WBT_STATE_ON_DEFAULT;  in wbt_init()
803 	rwb->wc = 1;  in wbt_init()
804 	rwb->rq_depth.default_depth = RWB_DEF_DEPTH;  in wbt_init()
805 	__wbt_update_limits(rwb);  in wbt_init()
810 	rq_qos_add(q, &rwb->rqos);  in wbt_init()
811 	blk_stat_add_callback(q, rwb->cb);  in wbt_init()
813 	rwb->min_lat_nsec = wbt_default_latency_nsec(q);  in wbt_init()