78 static inline bool rwb_enabled(struct rq_wb *rwb)  in rwb_enabled()  argument
80 	return rwb && rwb->wb_normal != 0;  in rwb_enabled()
83 static void wb_timestamp(struct rq_wb *rwb, unsigned long *var)  in wb_timestamp()  argument
85 	if (rwb_enabled(rwb)) {  in wb_timestamp()
97 static bool wb_recent_wait(struct rq_wb *rwb)  in wb_recent_wait()  argument
99 	struct bdi_writeback *wb = &rwb->rqos.q->backing_dev_info->wb;  in wb_recent_wait()
104 static inline struct rq_wait *get_rq_wait(struct rq_wb *rwb,  in get_rq_wait()  argument
108 		return &rwb->rq_wait[WBT_RWQ_KSWAPD];  in get_rq_wait()
110 		return &rwb->rq_wait[WBT_RWQ_DISCARD];  in get_rq_wait()
112 	return &rwb->rq_wait[WBT_RWQ_BG];  in get_rq_wait()
115 static void rwb_wake_all(struct rq_wb *rwb)  in rwb_wake_all()  argument
120 		struct rq_wait *rqw = &rwb->rq_wait[i];  in rwb_wake_all()
127 static void wbt_rqw_done(struct rq_wb *rwb, struct rq_wait *rqw,  in wbt_rqw_done()  argument
138 	if (unlikely(!rwb_enabled(rwb))) {  in wbt_rqw_done()
139 		rwb_wake_all(rwb);  in wbt_rqw_done()
149 		limit = rwb->wb_background;  in wbt_rqw_done()
150 	else if (rwb->wc && !wb_recent_wait(rwb))  in wbt_rqw_done()
153 		limit = rwb->wb_normal;  in wbt_rqw_done()
164 		if (!inflight || diff >= rwb->wb_background / 2)  in wbt_rqw_done()
171 	struct rq_wb *rwb = RQWB(rqos);  in __wbt_done()  local
177 	rqw = get_rq_wait(rwb, wb_acct);  in __wbt_done()
178 	wbt_rqw_done(rwb, rqw, wb_acct);  in __wbt_done()
187 	struct rq_wb *rwb = RQWB(rqos);  in wbt_done()  local
190 		if (rwb->sync_cookie == rq) {  in wbt_done()
191 			rwb->sync_issue = 0;  in wbt_done()
192 			rwb->sync_cookie = NULL;  in wbt_done()
196 			wb_timestamp(rwb, &rwb->last_comp);  in wbt_done()
198 		WARN_ON_ONCE(rq == rwb->sync_cookie);  in wbt_done()
216 static u64 rwb_sync_issue_lat(struct rq_wb *rwb)  in rwb_sync_issue_lat()  argument
218 	u64 now, issue = READ_ONCE(rwb->sync_issue);  in rwb_sync_issue_lat()
220 	if (!issue || !rwb->sync_cookie)  in rwb_sync_issue_lat()
234 static int latency_exceeded(struct rq_wb *rwb, struct blk_rq_stat *stat)  in latency_exceeded()  argument
236 	struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;  in latency_exceeded()
237 	struct rq_depth *rqd = &rwb->rq_depth;  in latency_exceeded()
249 	thislat = rwb_sync_issue_lat(rwb);  in latency_exceeded()
250 	if (thislat > rwb->cur_win_nsec ||  in latency_exceeded()
251 	    (thislat > rwb->min_lat_nsec && !stat[READ].nr_samples)) {  in latency_exceeded()
266 		if (stat[WRITE].nr_samples || wb_recent_wait(rwb) ||  in latency_exceeded()
267 		    wbt_inflight(rwb))  in latency_exceeded()
275 	if (stat[READ].min > rwb->min_lat_nsec) {  in latency_exceeded()
287 static void rwb_trace_step(struct rq_wb *rwb, const char *msg)  in rwb_trace_step()  argument
289 	struct backing_dev_info *bdi = rwb->rqos.q->backing_dev_info;  in rwb_trace_step()
290 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_trace_step()
292 	trace_wbt_step(bdi, msg, rqd->scale_step, rwb->cur_win_nsec,  in rwb_trace_step()
293 			rwb->wb_background, rwb->wb_normal, rqd->max_depth);  in rwb_trace_step()
296 static void calc_wb_limits(struct rq_wb *rwb)  in calc_wb_limits()  argument
298 	if (rwb->min_lat_nsec == 0) {  in calc_wb_limits()
299 		rwb->wb_normal = rwb->wb_background = 0;  in calc_wb_limits()
300 	} else if (rwb->rq_depth.max_depth <= 2) {  in calc_wb_limits()
301 		rwb->wb_normal = rwb->rq_depth.max_depth;  in calc_wb_limits()
302 		rwb->wb_background = 1;  in calc_wb_limits()
304 		rwb->wb_normal = (rwb->rq_depth.max_depth + 1) / 2;  in calc_wb_limits()
305 		rwb->wb_background = (rwb->rq_depth.max_depth + 3) / 4;  in calc_wb_limits()
309 static void scale_up(struct rq_wb *rwb)  in scale_up()  argument
311 	if (!rq_depth_scale_up(&rwb->rq_depth))  in scale_up()
313 	calc_wb_limits(rwb);  in scale_up()
314 	rwb->unknown_cnt = 0;  in scale_up()
315 	rwb_wake_all(rwb);  in scale_up()
316 	rwb_trace_step(rwb, "scale up");  in scale_up()
319 static void scale_down(struct rq_wb *rwb, bool hard_throttle)  in scale_down()  argument
321 	if (!rq_depth_scale_down(&rwb->rq_depth, hard_throttle))  in scale_down()
323 	calc_wb_limits(rwb);  in scale_down()
324 	rwb->unknown_cnt = 0;  in scale_down()
325 	rwb_trace_step(rwb, "scale down");  in scale_down()
328 static void rwb_arm_timer(struct rq_wb *rwb)  in rwb_arm_timer()  argument
330 	struct rq_depth *rqd = &rwb->rq_depth;  in rwb_arm_timer()
339 		rwb->cur_win_nsec = div_u64(rwb->win_nsec << 4,  in rwb_arm_timer()
346 		rwb->cur_win_nsec = rwb->win_nsec;  in rwb_arm_timer()
349 	blk_stat_activate_nsecs(rwb->cb, rwb->cur_win_nsec);  in rwb_arm_timer()
354 	struct rq_wb *rwb = cb->data;  in wb_timer_fn()  local
355 	struct rq_depth *rqd = &rwb->rq_depth;  in wb_timer_fn()
356 	unsigned int inflight = wbt_inflight(rwb);  in wb_timer_fn()
359 	status = latency_exceeded(rwb, cb->stat);  in wb_timer_fn()
361 	trace_wbt_timer(rwb->rqos.q->backing_dev_info, status, rqd->scale_step,  in wb_timer_fn()
371 		scale_down(rwb, true);  in wb_timer_fn()
374 		scale_up(rwb);  in wb_timer_fn()
382 		scale_up(rwb);  in wb_timer_fn()
385 		if (++rwb->unknown_cnt < RWB_UNKNOWN_BUMP)  in wb_timer_fn()
393 			scale_up(rwb);  in wb_timer_fn()
395 			scale_down(rwb, false);  in wb_timer_fn()
405 		rwb_arm_timer(rwb);  in wb_timer_fn()
408 static void __wbt_update_limits(struct rq_wb *rwb)  in __wbt_update_limits()  argument
410 	struct rq_depth *rqd = &rwb->rq_depth;  in __wbt_update_limits()
416 	calc_wb_limits(rwb);  in __wbt_update_limits()
418 	rwb_wake_all(rwb);  in __wbt_update_limits()
448 static bool close_io(struct rq_wb *rwb)  in close_io()  argument
452 	return time_before(now, rwb->last_issue + HZ / 10) ||  in close_io()
453 		time_before(now, rwb->last_comp + HZ / 10);  in close_io()
458 static inline unsigned int get_limit(struct rq_wb *rwb, unsigned long rw)  in get_limit()  argument
466 	if (!rwb_enabled(rwb))  in get_limit()
470 		return rwb->wb_background;  in get_limit()
480 	if ((rw & REQ_HIPRIO) || wb_recent_wait(rwb) || current_is_kswapd())  in get_limit()
481 		limit = rwb->rq_depth.max_depth;  in get_limit()
482 	else if ((rw & REQ_BACKGROUND) || close_io(rwb)) {  in get_limit()
487 		limit = rwb->wb_background;  in get_limit()
489 		limit = rwb->wb_normal;  in get_limit()
495 	struct rq_wb *rwb;  member
503 	return rq_wait_inc_below(rqw, get_limit(data->rwb, data->rw));  in wbt_inflight_cb()
509 	wbt_rqw_done(data->rwb, rqw, data->wb_acct);  in wbt_cleanup_cb()
516 static void __wbt_wait(struct rq_wb *rwb, enum wbt_flags wb_acct,  in __wbt_wait()  argument
519 	struct rq_wait *rqw = get_rq_wait(rwb, wb_acct);  in __wbt_wait()
521 		.rwb = rwb,  in __wbt_wait()
529 static inline bool wbt_should_throttle(struct rq_wb *rwb, struct bio *bio)  in wbt_should_throttle()  argument
547 static enum wbt_flags bio_to_wbt_flags(struct rq_wb *rwb, struct bio *bio)  in bio_to_wbt_flags()  argument
551 	if (!rwb_enabled(rwb))  in bio_to_wbt_flags()
556 	} else if (wbt_should_throttle(rwb, bio)) {  in bio_to_wbt_flags()
568 	struct rq_wb *rwb = RQWB(rqos);  in wbt_cleanup()  local
569 	enum wbt_flags flags = bio_to_wbt_flags(rwb, bio);  in wbt_cleanup()
581 	struct rq_wb *rwb = RQWB(rqos);  in wbt_wait()  local
584 	flags = bio_to_wbt_flags(rwb, bio);  in wbt_wait()
587 			wb_timestamp(rwb, &rwb->last_issue);  in wbt_wait()
591 	__wbt_wait(rwb, flags, bio->bi_opf);  in wbt_wait()
593 	if (!blk_stat_is_active(rwb->cb))  in wbt_wait()
594 		rwb_arm_timer(rwb);  in wbt_wait()
599 	struct rq_wb *rwb = RQWB(rqos);  in wbt_track()  local
600 	rq->wbt_flags |= bio_to_wbt_flags(rwb, bio);  in wbt_track()
605 	struct rq_wb *rwb = RQWB(rqos);  in wbt_issue()  local
607 	if (!rwb_enabled(rwb))  in wbt_issue()
617 	if (wbt_is_read(rq) && !rwb->sync_issue) {  in wbt_issue()
618 		rwb->sync_cookie = rq;  in wbt_issue()
619 		rwb->sync_issue = rq->io_start_time_ns;  in wbt_issue()
625 	struct rq_wb *rwb = RQWB(rqos);  in wbt_requeue()  local
626 	if (!rwb_enabled(rwb))  in wbt_requeue()
628 	if (rq == rwb->sync_cookie) {  in wbt_requeue()
629 		rwb->sync_issue = 0;  in wbt_requeue()
630 		rwb->sync_cookie = NULL;  in wbt_requeue()
693 	struct rq_wb *rwb = RQWB(rqos);  in wbt_exit()  local
696 	blk_stat_remove_callback(q, rwb->cb);  in wbt_exit()
697 	blk_stat_free_callback(rwb->cb);  in wbt_exit()
698 	kfree(rwb);  in wbt_exit()
707 	struct rq_wb *rwb;  in wbt_disable_default()  local
710 	rwb = RQWB(rqos);  in wbt_disable_default()
711 	if (rwb->enable_state == WBT_STATE_ON_DEFAULT) {  in wbt_disable_default()
712 		blk_stat_deactivate(rwb->cb);  in wbt_disable_default()
713 		rwb->wb_normal = 0;  in wbt_disable_default()
722 	struct rq_wb *rwb = RQWB(rqos);  in wbt_curr_win_nsec_show()  local
724 	seq_printf(m, "%llu\n", rwb->cur_win_nsec);  in wbt_curr_win_nsec_show()
731 	struct rq_wb *rwb = RQWB(rqos);  in wbt_enabled_show()  local
733 	seq_printf(m, "%d\n", rwb->enable_state);  in wbt_enabled_show()
748 	struct rq_wb *rwb = RQWB(rqos);  in wbt_inflight_show()  local
753 			   atomic_read(&rwb->rq_wait[i].inflight));  in wbt_inflight_show()
760 	struct rq_wb *rwb = RQWB(rqos);  in wbt_min_lat_nsec_show()  local
762 	seq_printf(m, "%lu\n", rwb->min_lat_nsec);  in wbt_min_lat_nsec_show()
769 	struct rq_wb *rwb = RQWB(rqos);  in wbt_unknown_cnt_show()  local
771 	seq_printf(m, "%u\n", rwb->unknown_cnt);  in wbt_unknown_cnt_show()
778 	struct rq_wb *rwb = RQWB(rqos);  in wbt_normal_show()  local
780 	seq_printf(m, "%u\n", rwb->wb_normal);  in wbt_normal_show()
787 	struct rq_wb *rwb = RQWB(rqos);  in wbt_background_show()  local
789 	seq_printf(m, "%u\n", rwb->wb_background);  in wbt_background_show()
822 	struct rq_wb *rwb;  in wbt_init()  local
825 	rwb = kzalloc(sizeof(*rwb), GFP_KERNEL);  in wbt_init()
826 	if (!rwb)  in wbt_init()
829 	rwb->cb = blk_stat_alloc_callback(wb_timer_fn, wbt_data_dir, 2, rwb);  in wbt_init()
830 	if (!rwb->cb) {  in wbt_init()
831 		kfree(rwb);  in wbt_init()
836 		rq_wait_init(&rwb->rq_wait[i]);  in wbt_init()
838 	rwb->rqos.id = RQ_QOS_WBT;  in wbt_init()
839 	rwb->rqos.ops = &wbt_rqos_ops;  in wbt_init()
840 	rwb->rqos.q = q;  in wbt_init()
841 	rwb->last_comp = rwb->last_issue = jiffies;  in wbt_init()
842 	rwb->win_nsec = RWB_WINDOW_NSEC;  in wbt_init()
843 	rwb->enable_state = WBT_STATE_ON_DEFAULT;  in wbt_init()
844 	rwb->wc = 1;  in wbt_init()
845 	rwb->rq_depth.default_depth = RWB_DEF_DEPTH;  in wbt_init()
846 	__wbt_update_limits(rwb);  in wbt_init()
851 	rq_qos_add(q, &rwb->rqos);  in wbt_init()
852 	blk_stat_add_callback(q, rwb->cb);  in wbt_init()
854 	rwb->min_lat_nsec = wbt_default_latency_nsec(q);  in wbt_init()
856 	wbt_queue_depth_changed(&rwb->rqos);  in wbt_init()