54 	struct bdi_writeback *wb = &bdi->wb;  in bdi_debug_stats_show()  local
62 	spin_lock(&wb->list_lock);  in bdi_debug_stats_show()
63 	list_for_each_entry(inode, &wb->b_dirty, i_io_list)  in bdi_debug_stats_show()
65 	list_for_each_entry(inode, &wb->b_io, i_io_list)  in bdi_debug_stats_show()
67 	list_for_each_entry(inode, &wb->b_more_io, i_io_list)  in bdi_debug_stats_show()
69 	list_for_each_entry(inode, &wb->b_dirty_time, i_io_list)  in bdi_debug_stats_show()
72 	spin_unlock(&wb->list_lock);  in bdi_debug_stats_show()
75 	wb_thresh = wb_calc_thresh(wb, dirty_thresh);  in bdi_debug_stats_show()
92 		   (unsigned long) K(wb_stat(wb, WB_WRITEBACK)),  in bdi_debug_stats_show()
93 		   (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)),  in bdi_debug_stats_show()
97 		   (unsigned long) K(wb_stat(wb, WB_DIRTIED)),  in bdi_debug_stats_show()
98 		   (unsigned long) K(wb_stat(wb, WB_WRITTEN)),  in bdi_debug_stats_show()
99 		   (unsigned long) K(wb->write_bandwidth),  in bdi_debug_stats_show()
104 		   !list_empty(&bdi->bdi_list), bdi->wb.state);  in bdi_debug_stats_show()
258 void wb_wakeup_delayed(struct bdi_writeback *wb)  in wb_wakeup_delayed()  argument
263 	spin_lock_irq(&wb->work_lock);  in wb_wakeup_delayed()
264 	if (test_bit(WB_registered, &wb->state))  in wb_wakeup_delayed()
265 		queue_delayed_work(bdi_wq, &wb->dwork, timeout);  in wb_wakeup_delayed()
266 	spin_unlock_irq(&wb->work_lock);  in wb_wakeup_delayed()
271 	struct bdi_writeback *wb = container_of(to_delayed_work(work),  in wb_update_bandwidth_workfn()  local
274 	wb_update_bandwidth(wb);  in wb_update_bandwidth_workfn()
282 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi,  in wb_init()  argument
287 	memset(wb, 0, sizeof(*wb));  in wb_init()
289 	wb->bdi = bdi;  in wb_init()
290 	wb->last_old_flush = jiffies;  in wb_init()
291 	INIT_LIST_HEAD(&wb->b_dirty);  in wb_init()
292 	INIT_LIST_HEAD(&wb->b_io);  in wb_init()
293 	INIT_LIST_HEAD(&wb->b_more_io);  in wb_init()
294 	INIT_LIST_HEAD(&wb->b_dirty_time);  in wb_init()
295 	spin_lock_init(&wb->list_lock);  in wb_init()
297 	atomic_set(&wb->writeback_inodes, 0);  in wb_init()
298 	wb->bw_time_stamp = jiffies;  in wb_init()
299 	wb->balanced_dirty_ratelimit = INIT_BW;  in wb_init()
300 	wb->dirty_ratelimit = INIT_BW;  in wb_init()
301 	wb->write_bandwidth = INIT_BW;  in wb_init()
302 	wb->avg_write_bandwidth = INIT_BW;  in wb_init()
304 	spin_lock_init(&wb->work_lock);  in wb_init()
305 	INIT_LIST_HEAD(&wb->work_list);  in wb_init()
306 	INIT_DELAYED_WORK(&wb->dwork, wb_workfn);  in wb_init()
307 	INIT_DELAYED_WORK(&wb->bw_dwork, wb_update_bandwidth_workfn);  in wb_init()
308 	wb->dirty_sleep = jiffies;  in wb_init()
310 	err = fprop_local_init_percpu(&wb->completions, gfp);  in wb_init()
315 		err = percpu_counter_init(&wb->stat[i], 0, gfp);  in wb_init()
324 		percpu_counter_destroy(&wb->stat[i]);  in wb_init()
325 	fprop_local_destroy_percpu(&wb->completions);  in wb_init()
329 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb);
334 static void wb_shutdown(struct bdi_writeback *wb)  in wb_shutdown()  argument
337 	spin_lock_irq(&wb->work_lock);  in wb_shutdown()
338 	if (!test_and_clear_bit(WB_registered, &wb->state)) {  in wb_shutdown()
339 		spin_unlock_irq(&wb->work_lock);  in wb_shutdown()
342 	spin_unlock_irq(&wb->work_lock);  in wb_shutdown()
344 	cgwb_remove_from_bdi_list(wb);  in wb_shutdown()
350 	mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_shutdown()
351 	flush_delayed_work(&wb->dwork);  in wb_shutdown()
352 	WARN_ON(!list_empty(&wb->work_list));  in wb_shutdown()
353 	flush_delayed_work(&wb->bw_dwork);  in wb_shutdown()
356 static void wb_exit(struct bdi_writeback *wb)  in wb_exit()  argument
360 	WARN_ON(delayed_work_pending(&wb->dwork));  in wb_exit()
363 		percpu_counter_destroy(&wb->stat[i]);  in wb_exit()
365 	fprop_local_destroy_percpu(&wb->completions);  in wb_exit()
385 	struct bdi_writeback *wb = container_of(work, struct bdi_writeback,  in cgwb_release_workfn()  local
387 	struct backing_dev_info *bdi = wb->bdi;  in cgwb_release_workfn()
389 	mutex_lock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
390 	wb_shutdown(wb);  in cgwb_release_workfn()
392 	css_put(wb->memcg_css);  in cgwb_release_workfn()
393 	css_put(wb->blkcg_css);  in cgwb_release_workfn()
394 	mutex_unlock(&wb->bdi->cgwb_release_mutex);  in cgwb_release_workfn()
397 	blkcg_unpin_online(wb->blkcg_css);  in cgwb_release_workfn()
399 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_release_workfn()
402 	list_del(&wb->offline_node);  in cgwb_release_workfn()
405 	percpu_ref_exit(&wb->refcnt);  in cgwb_release_workfn()
406 	wb_exit(wb);  in cgwb_release_workfn()
408 	WARN_ON_ONCE(!list_empty(&wb->b_attached));  in cgwb_release_workfn()
409 	kfree_rcu(wb, rcu);  in cgwb_release_workfn()
414 	struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback,  in cgwb_release()  local
416 	queue_work(cgwb_release_wq, &wb->release_work);  in cgwb_release()
419 static void cgwb_kill(struct bdi_writeback *wb)  in cgwb_kill()  argument
423 	WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id));  in cgwb_kill()
424 	list_del(&wb->memcg_node);  in cgwb_kill()
425 	list_del(&wb->blkcg_node);  in cgwb_kill()
426 	list_add(&wb->offline_node, &offline_cgwbs);  in cgwb_kill()
427 	percpu_ref_kill(&wb->refcnt);  in cgwb_kill()
430 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
433 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
443 	struct bdi_writeback *wb;  in cgwb_create()  local
454 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in cgwb_create()
455 	if (wb && wb->blkcg_css != blkcg_css) {  in cgwb_create()
456 		cgwb_kill(wb);  in cgwb_create()
457 		wb = NULL;  in cgwb_create()
460 	if (wb)  in cgwb_create()
464 	wb = kmalloc(sizeof(*wb), gfp);  in cgwb_create()
465 	if (!wb) {  in cgwb_create()
470 	ret = wb_init(wb, bdi, gfp);  in cgwb_create()
474 	ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp);  in cgwb_create()
478 	ret = fprop_local_init_percpu(&wb->memcg_completions, gfp);  in cgwb_create()
482 	wb->memcg_css = memcg_css;  in cgwb_create()
483 	wb->blkcg_css = blkcg_css;  in cgwb_create()
484 	INIT_LIST_HEAD(&wb->b_attached);  in cgwb_create()
485 	INIT_WORK(&wb->release_work, cgwb_release_workfn);  in cgwb_create()
486 	set_bit(WB_registered, &wb->state);  in cgwb_create()
497 	if (test_bit(WB_registered, &bdi->wb.state) &&  in cgwb_create()
500 		ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb);  in cgwb_create()
502 			list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list);  in cgwb_create()
503 			list_add(&wb->memcg_node, memcg_cgwb_list);  in cgwb_create()
504 			list_add(&wb->blkcg_node, blkcg_cgwb_list);  in cgwb_create()
520 	fprop_local_destroy_percpu(&wb->memcg_completions);  in cgwb_create()
522 	percpu_ref_exit(&wb->refcnt);  in cgwb_create()
524 	wb_exit(wb);  in cgwb_create()
526 	kfree(wb);  in cgwb_create()
558 	struct bdi_writeback *wb;  in wb_get_lookup()  local
561 		return &bdi->wb;  in wb_get_lookup()
564 	wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id);  in wb_get_lookup()
565 	if (wb) {  in wb_get_lookup()
570 		if (unlikely(wb->blkcg_css != blkcg_css || !wb_tryget(wb)))  in wb_get_lookup()
571 			wb = NULL;  in wb_get_lookup()
576 	return wb;  in wb_get_lookup()
592 	struct bdi_writeback *wb;  in wb_get_create()  local
597 		return &bdi->wb;  in wb_get_create()
600 		wb = wb_get_lookup(bdi, memcg_css);  in wb_get_create()
601 	} while (!wb && !cgwb_create(bdi, memcg_css, gfp));  in wb_get_create()
603 	return wb;  in wb_get_create()
614 	ret = wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
616 		bdi->wb.memcg_css = &root_mem_cgroup->css;  in cgwb_bdi_init()
617 		bdi->wb.blkcg_css = blkcg_root_css;  in cgwb_bdi_init()
626 	struct bdi_writeback *wb;  in cgwb_bdi_unregister()  local
628 	WARN_ON(test_bit(WB_registered, &bdi->wb.state));  in cgwb_bdi_unregister()
638 		wb = list_first_entry(&bdi->wb_list, struct bdi_writeback,  in cgwb_bdi_unregister()
641 		wb_shutdown(wb);  in cgwb_bdi_unregister()
657 	struct bdi_writeback *wb;  in cleanup_offline_cgwbs_workfn()  local
663 		wb = list_first_entry(&offline_cgwbs, struct bdi_writeback,  in cleanup_offline_cgwbs_workfn()
665 		list_move(&wb->offline_node, &processed);  in cleanup_offline_cgwbs_workfn()
676 		if (wb_has_dirty_io(wb))  in cleanup_offline_cgwbs_workfn()
679 		if (!wb_tryget(wb))  in cleanup_offline_cgwbs_workfn()
683 		while (cleanup_offline_cgwb(wb))  in cleanup_offline_cgwbs_workfn()
687 		wb_put(wb);  in cleanup_offline_cgwbs_workfn()
705 	struct bdi_writeback *wb, *next;  in wb_memcg_offline()  local
708 	list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node)  in wb_memcg_offline()
709 		cgwb_kill(wb);  in wb_memcg_offline()
724 	struct bdi_writeback *wb, *next;  in wb_blkcg_offline()  local
728 	list_for_each_entry_safe(wb, next, list, blkcg_node)  in wb_blkcg_offline()
729 		cgwb_kill(wb);  in wb_blkcg_offline()
737 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
760 	return wb_init(&bdi->wb, bdi, GFP_KERNEL);  in cgwb_bdi_init()
767 	list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list);  in cgwb_bdi_register()
770 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb)  in cgwb_remove_from_bdi_list()  argument
772 	list_del_rcu(&wb->bdi_node);  in cgwb_remove_from_bdi_list()
877 	set_bit(WB_registered, &bdi->wb.state);  in bdi_register_va()
933 	wb_shutdown(&bdi->wb);  in bdi_unregister()
961 	WARN_ON_ONCE(test_bit(WB_registered, &bdi->wb.state));  in release_bdi()
963 	wb_exit(&bdi->wb);  in release_bdi()