51 	struct bdi_writeback *wb = &bdi->wb;  in bdi_debug_stats_show()  local
59 	spin_lock(&wb->list_lock);  in bdi_debug_stats_show()
60 	list_for_each_entry(inode, &wb->b_dirty, i_io_list)  in bdi_debug_stats_show()
62 	list_for_each_entry(inode, &wb->b_io, i_io_list)  in bdi_debug_stats_show()
64 	list_for_each_entry(inode, &wb->b_more_io, i_io_list)  in bdi_debug_stats_show()
66 	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)  in bdi_debug_stats_show()
69 	spin_unlock(&wb->list_lock);  in bdi_debug_stats_show()
72 	wb_thresh = wb_calc_thresh(wb, dirty_thresh);  in bdi_debug_stats_show()
90 		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),  in bdi_debug_stats_show()
91 		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),  in bdi_debug_stats_show()
95 		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),  in bdi_debug_stats_show()
96 		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),  in bdi_debug_stats_show()
97 		   (unsigned long) K(wb->write_bandwidth),  in bdi_debug_stats_show()
102 		   !list_empty(&bdi->bdi_list), bdi->wb.state);  in bdi_debug_stats_show()
267 void wb_wakeup_delayed(struct bdi_writeback *wb)  in wb_wakeup_delayed()  argument
272 	spin_lock_bh(&wb->work_lock);  in wb_wakeup_delayed()
273 	if (test_bit(WB_registered, &wb->state))  in wb_wakeup_delayed()
274 		queue_delayed_work(bdi_wq, &wb->dwork, timeout);  in wb_wakeup_delayed()
275 	spin_unlock_bh(&wb->work_lock);  in wb_wakeup_delayed()
283 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,  in wb_init()  argument
288 	memset(wb, 0, sizeof(*wb));  in wb_init()
290 	if (wb != &bdi->wb)  in wb_init()
292 	wb->bdi = bdi;  in wb_init()
293 	wb->last_old_flush = jiffies;  in wb_init()
294 	INIT_LIST_HEAD(&wb->b_dirty);  in wb_init()
295 	INIT_LIST_HEAD(&wb->b_io);  in wb_init()
296 	INIT_LIST_HEAD(&wb->b_more_io);  in wb_init()
297 	INIT_LIST_HEAD(&wb->b_dirty_time);  in wb_init()
298 	spin_lock_init(&wb->list_lock);  in wb_init()
300 	wb->bw_time_stamp = jiffies;  in wb_init()
301 	wb->balanced_dirty_ratelimit = INIT_BW;  in wb_init()
302 	wb->dirty_ratelimit = INIT_BW;  in wb_init()
303 	wb->write_bandwidth = INIT_BW;  in wb_init()
304 	wb->avg_write_bandwidth = INIT_BW;  in wb_init()
306 	spin_lock_init(&wb->work_lock);  in wb_init()
307 	INIT_LIST_HEAD(&wb->work_list);  in wb_init()
308 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);  in wb_init()
309 	wb->dirty_sleep = jiffies;  in wb_init()
311 	wb->congested = wb_congested_get_create(bdi, blkcg_id, gfp);  in wb_init()
312 	if (!wb->congested) {  in wb_init()
317 	err = fprop_local_init_percpu(&wb->completions, gfp);  in wb_init()
322 		err = percpu_counter_init(&wb->stat[i], 0, gfp);  in wb_init()
331 		percpu_counter_destroy(&wb->stat[i]);  in wb_init()
332 	fprop_local_destroy_percpu(&wb->completions);  in wb_init()
334 	wb_congested_put(wb->congested);  in wb_init()
336 	if (wb != &bdi->wb)  in wb_init()
341 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb);
346 static void wb_shutdown(struct bdi_writeback *wb)  in wb_shutdown()  argument
349 	spin_lock_bh(&wb->work_lock);  in wb_shutdown()
350 	if (!test_and_clear_bit(WB_registered, &wb->state)) {  in wb_shutdown()
351 		spin_unlock_bh(&wb->work_lock);  in wb_shutdown()
354 	spin_unlock_bh(&wb->work_lock);  in wb_shutdown()
356 	cgwb_remove_from_bdi_list(wb);  in wb_shutdown()
362 	mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_shutdown()
363 	flush_delayed_work(&wb->dwork);  in wb_shutdown()
364 	WARN_ON(!list_empty(&wb->work_list));  in wb_shutdown()
367 static void wb_exit(struct bdi_writeback *wb)  in wb_exit()  argument
371 	WARN_ON(delayed_work_pending(&wb->dwork));  in wb_exit()
374 		percpu_counter_destroy(&wb->stat[i]);  in wb_exit()
376 	fprop_local_destroy_percpu(&wb->completions);  in wb_exit()
377 	wb_congested_put(wb->congested);  in wb_exit()
378 	if (wb != &wb->bdi->wb)  in wb_exit()
379 		bdi_put(wb->bdi);  in wb_exit()
482 	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,  in cgwb_release_workfn()  local
484 	struct blkcg *blkcg = css_to_blkcg(wb->blkcg_css);  in cgwb_release_workfn()
486 	mutex_lock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
487 	wb_shutdown(wb);  in cgwb_release_workfn()
489 	css_put(wb->memcg_css);  in cgwb_release_workfn()
490 	css_put(wb->blkcg_css);  in cgwb_release_workfn()
491 	mutex_unlock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
496 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_release_workfn()
497 	percpu_ref_exit(&wb->refcnt);  in cgwb_release_workfn()
498 	wb_exit(wb);  in cgwb_release_workfn()
499 	kfree_rcu(wb, rcu);  in cgwb_release_workfn()
504 	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,  in cgwb_release()  local
506 	queue_work(cgwb_release_wq, &wb->release_work);  in cgwb_release()
509 static void cgwb_kill(struct bdi_writeback *wb)  in cgwb_kill()  argument
513 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));  in cgwb_kill()
514 	list_del(&wb->memcg_node);  in cgwb_kill()
515 	list_del(&wb->blkcg_node);  in cgwb_kill()
516 	percpu_ref_kill(&wb->refcnt);  in cgwb_kill()
519 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
522 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
533 	struct bdi_writeback *wb;  in cgwb_create()  local
545 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in cgwb_create()
546 	if (wb && wb->blkcg_css != blkcg_css) {  in cgwb_create()
547 		cgwb_kill(wb);  in cgwb_create()
548 		wb = NULL;  in cgwb_create()
551 	if (wb)  in cgwb_create()
555 	wb = kmalloc(sizeof(*wb), gfp);  in cgwb_create()
556 	if (!wb) {  in cgwb_create()
561 	ret = wb_init(wb, bdi, blkcg_css->id, gfp);  in cgwb_create()
565 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);  in cgwb_create()
569 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);  in cgwb_create()
573 	wb->memcg_css = memcg_css;  in cgwb_create()
574 	wb->blkcg_css = blkcg_css;  in cgwb_create()
575 	INIT_WORK(&wb->release_work, cgwb_release_workfn);  in cgwb_create()
576 	set_bit(WB_registered, &wb->state);  in cgwb_create()
586 	if (test_bit(WB_registered, &bdi->wb.state) &&  in cgwb_create()
589 		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);  in cgwb_create()
591 			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);  in cgwb_create()
592 			list_add(&wb->memcg_node, memcg_cgwb_list);  in cgwb_create()
593 			list_add(&wb->blkcg_node, blkcg_cgwb_list);  in cgwb_create()
608 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_create()
610 	percpu_ref_exit(&wb->refcnt);  in cgwb_create()
612 	wb_exit(wb);  in cgwb_create()
614 	kfree(wb);  in cgwb_create()
646 	struct bdi_writeback *wb;  in wb_get_lookup()  local
649 		return &bdi->wb;  in wb_get_lookup()
652 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in wb_get_lookup()
653 	if (wb) {  in wb_get_lookup()
658 		if (unlikely(wb->blkcg_css != blkcg_css || !wb_tryget(wb)))  in wb_get_lookup()
659 			wb = NULL;  in wb_get_lookup()
664 	return wb;  in wb_get_lookup()
680 	struct bdi_writeback *wb;  in wb_get_create()  local
685 		return &bdi->wb;  in wb_get_create()
688 		wb = wb_get_lookup(bdi, memcg_css);  in wb_get_create()
689 	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));  in wb_get_create()
691 	return wb;  in wb_get_create()
703 	ret = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);  in cgwb_bdi_init()
705 		bdi->wb.memcg_css = &root_mem_cgroup->css;  in cgwb_bdi_init()
706 		bdi->wb.blkcg_css = blkcg_root_css;  in cgwb_bdi_init()
715 	struct bdi_writeback *wb;  in cgwb_bdi_unregister()  local
717 	WARN_ON(test_bit(WB_registered, &bdi->wb.state));  in cgwb_bdi_unregister()
727 		wb = list_first_entry(&bdi->wb_list, struct bdi_writeback,  in cgwb_bdi_unregister()
730 		wb_shutdown(wb);  in cgwb_bdi_unregister()
746 	struct bdi_writeback *wb, *next;  in wb_memcg_offline()  local
749 	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)  in wb_memcg_offline()
750 		cgwb_kill(wb);  in wb_memcg_offline()
763 	struct bdi_writeback *wb, *next;  in wb_blkcg_offline()  local
766 	list_for_each_entry_safe(wb, next, &blkcg->cgwb_list, blkcg_node)  in wb_blkcg_offline()
767 		cgwb_kill(wb);  in wb_blkcg_offline()
790 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
821 	err = wb_init(&bdi->wb, bdi, 1, GFP_KERNEL);  in cgwb_bdi_init()
838 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
841 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
843 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
948 	set_bit(WB_registered, &bdi->wb.state);  in bdi_register_va()
1011 	wb_shutdown(&bdi->wb);  in bdi_unregister()
1031 	if (test_bit(WB_registered, &bdi->wb.state))  in release_bdi()
1034 	wb_exit(&bdi->wb);  in release_bdi()