52 	struct bdi_writeback *wb = &bdi->wb;  in bdi_debug_stats_show()  local
60 	spin_lock(&wb->list_lock);  in bdi_debug_stats_show()
61 	list_for_each_entry(inode, &wb->b_dirty, i_io_list)  in bdi_debug_stats_show()
63 	list_for_each_entry(inode, &wb->b_io, i_io_list)  in bdi_debug_stats_show()
65 	list_for_each_entry(inode, &wb->b_more_io, i_io_list)  in bdi_debug_stats_show()
67 	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)  in bdi_debug_stats_show()
70 	spin_unlock(&wb->list_lock);  in bdi_debug_stats_show()
73 	wb_thresh = wb_calc_thresh(wb, dirty_thresh);  in bdi_debug_stats_show()
90 		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),  in bdi_debug_stats_show()
91 		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),  in bdi_debug_stats_show()
95 		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),  in bdi_debug_stats_show()
96 		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),  in bdi_debug_stats_show()
97 		   (unsigned long) K(wb->write_bandwidth),  in bdi_debug_stats_show()
102 		   !list_empty(&bdi->bdi_list), bdi->wb.state);  in bdi_debug_stats_show()
263 void wb_wakeup_delayed(struct bdi_writeback *wb)  in wb_wakeup_delayed()  argument
268 	spin_lock_bh(&wb->work_lock);  in wb_wakeup_delayed()
269 	if (test_bit(WB_registered, &wb->state))  in wb_wakeup_delayed()
270 		queue_delayed_work(bdi_wq, &wb->dwork, timeout);  in wb_wakeup_delayed()
271 	spin_unlock_bh(&wb->work_lock);  in wb_wakeup_delayed()
276 	struct bdi_writeback *wb = container_of(to_delayed_work(work),  in wb_update_bandwidth_workfn()  local
279 	wb_update_bandwidth(wb);  in wb_update_bandwidth_workfn()
287 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,  in wb_init()  argument
292 	memset(wb, 0, sizeof(*wb));  in wb_init()
294 	if (wb != &bdi->wb)  in wb_init()
296 	wb->bdi = bdi;  in wb_init()
297 	wb->last_old_flush = jiffies;  in wb_init()
298 	INIT_LIST_HEAD(&wb->b_dirty);  in wb_init()
299 	INIT_LIST_HEAD(&wb->b_io);  in wb_init()
300 	INIT_LIST_HEAD(&wb->b_more_io);  in wb_init()
301 	INIT_LIST_HEAD(&wb->b_dirty_time);  in wb_init()
302 	spin_lock_init(&wb->list_lock);  in wb_init()
304 	atomic_set(&wb->writeback_inodes, 0);  in wb_init()
305 	wb->bw_time_stamp = jiffies;  in wb_init()
306 	wb->balanced_dirty_ratelimit = INIT_BW;  in wb_init()
307 	wb->dirty_ratelimit = INIT_BW;  in wb_init()
308 	wb->write_bandwidth = INIT_BW;  in wb_init()
309 	wb->avg_write_bandwidth = INIT_BW;  in wb_init()
311 	spin_lock_init(&wb->work_lock);  in wb_init()
312 	INIT_LIST_HEAD(&wb->work_list);  in wb_init()
313 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);  in wb_init()
314 	INIT_DELAYED_WORK(&wb->bw_dwork, wb_update_bandwidth_workfn);  in wb_init()
315 	wb->dirty_sleep = jiffies;  in wb_init()
317 	err = fprop_local_init_percpu(&wb->completions, gfp);  in wb_init()
322 		err = percpu_counter_init(&wb->stat[i], 0, gfp);  in wb_init()
331 		percpu_counter_destroy(&wb->stat[i]);  in wb_init()
332 	fprop_local_destroy_percpu(&wb->completions);  in wb_init()
334 	if (wb != &bdi->wb)  in wb_init()
339 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb);
344 static void wb_shutdown(struct bdi_writeback *wb)  in wb_shutdown()  argument
347 	spin_lock_bh(&wb->work_lock);  in wb_shutdown()
348 	if (!test_and_clear_bit(WB_registered, &wb->state)) {  in wb_shutdown()
349 		spin_unlock_bh(&wb->work_lock);  in wb_shutdown()
352 	spin_unlock_bh(&wb->work_lock);  in wb_shutdown()
354 	cgwb_remove_from_bdi_list(wb);  in wb_shutdown()
360 	mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_shutdown()
361 	flush_delayed_work(&wb->dwork);  in wb_shutdown()
362 	WARN_ON(!list_empty(&wb->work_list));  in wb_shutdown()
363 	flush_delayed_work(&wb->bw_dwork);  in wb_shutdown()
366 static void wb_exit(struct bdi_writeback *wb)  in wb_exit()  argument
370 	WARN_ON(delayed_work_pending(&wb->dwork));  in wb_exit()
373 		percpu_counter_destroy(&wb->stat[i]);  in wb_exit()
375 	fprop_local_destroy_percpu(&wb->completions);  in wb_exit()
376 	if (wb != &wb->bdi->wb)  in wb_exit()
377 		bdi_put(wb->bdi);  in wb_exit()
397 	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,  in cgwb_release_workfn()  local
399 	struct blkcg *blkcg = css_to_blkcg(wb->blkcg_css);  in cgwb_release_workfn()
401 	mutex_lock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
402 	wb_shutdown(wb);  in cgwb_release_workfn()
404 	css_put(wb->memcg_css);  in cgwb_release_workfn()
405 	css_put(wb->blkcg_css);  in cgwb_release_workfn()
406 	mutex_unlock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
411 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_release_workfn()
414 	list_del(&wb->offline_node);  in cgwb_release_workfn()
417 	percpu_ref_exit(&wb->refcnt);  in cgwb_release_workfn()
418 	wb_exit(wb);  in cgwb_release_workfn()
419 	WARN_ON_ONCE(!list_empty(&wb->b_attached));  in cgwb_release_workfn()
420 	kfree_rcu(wb, rcu);  in cgwb_release_workfn()
425 	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,  in cgwb_release()  local
427 	queue_work(cgwb_release_wq, &wb->release_work);  in cgwb_release()
430 static void cgwb_kill(struct bdi_writeback *wb)  in cgwb_kill()  argument
434 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));  in cgwb_kill()
435 	list_del(&wb->memcg_node);  in cgwb_kill()
436 	list_del(&wb->blkcg_node);  in cgwb_kill()
437 	list_add(&wb->offline_node, &offline_cgwbs);  in cgwb_kill()
438 	percpu_ref_kill(&wb->refcnt);  in cgwb_kill()
441 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
444 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
455 	struct bdi_writeback *wb;  in cgwb_create()  local
467 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in cgwb_create()
468 	if (wb && wb->blkcg_css != blkcg_css) {  in cgwb_create()
469 		cgwb_kill(wb);  in cgwb_create()
470 		wb = NULL;  in cgwb_create()
473 	if (wb)  in cgwb_create()
477 	wb = kmalloc(sizeof(*wb), gfp);  in cgwb_create()
478 	if (!wb) {  in cgwb_create()
483 	ret = wb_init(wb, bdi, gfp);  in cgwb_create()
487 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);  in cgwb_create()
491 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);  in cgwb_create()
495 	wb->memcg_css = memcg_css;  in cgwb_create()
496 	wb->blkcg_css = blkcg_css;  in cgwb_create()
497 	INIT_LIST_HEAD(&wb->b_attached);  in cgwb_create()
498 	INIT_WORK(&wb->release_work, cgwb_release_workfn);  in cgwb_create()
499 	set_bit(WB_registered, &wb->state);  in cgwb_create()
509 	if (test_bit(WB_registered, &bdi->wb.state) &&  in cgwb_create()
512 		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);  in cgwb_create()
514 			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);  in cgwb_create()
515 			list_add(&wb->memcg_node, memcg_cgwb_list);  in cgwb_create()
516 			list_add(&wb->blkcg_node, blkcg_cgwb_list);  in cgwb_create()
531 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_create()
533 	percpu_ref_exit(&wb->refcnt);  in cgwb_create()
535 	wb_exit(wb);  in cgwb_create()
537 	kfree(wb);  in cgwb_create()
569 	struct bdi_writeback *wb;  in wb_get_lookup()  local
572 		return &bdi->wb;  in wb_get_lookup()
575 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in wb_get_lookup()
576 	if (wb) {  in wb_get_lookup()
581 		if (unlikely(wb->blkcg_css != blkcg_css || !wb_tryget(wb)))  in wb_get_lookup()
582 			wb = NULL;  in wb_get_lookup()
587 	return wb;  in wb_get_lookup()
603 	struct bdi_writeback *wb;  in wb_get_create()  local
608 		return &bdi->wb;  in wb_get_create()
611 		wb = wb_get_lookup(bdi, memcg_css);  in wb_get_create()
612 	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));  in wb_get_create()
614 	return wb;  in wb_get_create()
625 	ret = wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
627 		bdi->wb.memcg_css = &root_mem_cgroup->css;  in cgwb_bdi_init()
628 		bdi->wb.blkcg_css = blkcg_root_css;  in cgwb_bdi_init()
637 	struct bdi_writeback *wb;  in cgwb_bdi_unregister()  local
639 	WARN_ON(test_bit(WB_registered, &bdi->wb.state));  in cgwb_bdi_unregister()
649 		wb = list_first_entry(&bdi->wb_list, struct bdi_writeback,  in cgwb_bdi_unregister()
652 		wb_shutdown(wb);  in cgwb_bdi_unregister()
668 	struct bdi_writeback *wb;  in cleanup_offline_cgwbs_workfn()  local
674 		wb = list_first_entry(&offline_cgwbs, struct bdi_writeback,  in cleanup_offline_cgwbs_workfn()
676 		list_move(&wb->offline_node, &processed);  in cleanup_offline_cgwbs_workfn()
687 		if (wb_has_dirty_io(wb))  in cleanup_offline_cgwbs_workfn()
690 		if (!wb_tryget(wb))  in cleanup_offline_cgwbs_workfn()
694 		while (cleanup_offline_cgwb(wb))  in cleanup_offline_cgwbs_workfn()
698 		wb_put(wb);  in cleanup_offline_cgwbs_workfn()
716 	struct bdi_writeback *wb, *next;  in wb_memcg_offline()  local
719 	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)  in wb_memcg_offline()
720 		cgwb_kill(wb);  in wb_memcg_offline()
735 	struct bdi_writeback *wb, *next;  in wb_blkcg_offline()  local
738 	list_for_each_entry_safe(wb, next, &blkcg->cgwb_list, blkcg_node)  in wb_blkcg_offline()
739 		cgwb_kill(wb);  in wb_blkcg_offline()
747 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
770 	return wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
777 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
780 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
782 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
891 	set_bit(WB_registered, &bdi->wb.state);  in bdi_register_va()
947 	wb_shutdown(&bdi->wb);  in bdi_unregister()
967 	if (test_bit(WB_registered, &bdi->wb.state))  in release_bdi()
970 	wb_exit(&bdi->wb);  in release_bdi()
1000 	if (test_and_clear_bit(bit, &bdi->wb.congested))  in clear_bdi_congested()
1013 	if (!test_and_set_bit(bit, &bdi->wb.congested))  in set_bdi_congested()