85 static bool wb_io_lists_populated(struct bdi_writeback *wb)  in wb_io_lists_populated()  argument
87 	if (wb_has_dirty_io(wb)) {  in wb_io_lists_populated()
90 		set_bit(WB_has_dirty_io, &wb->state);  in wb_io_lists_populated()
91 		WARN_ON_ONCE(!wb->avg_write_bandwidth);  in wb_io_lists_populated()
92 		atomic_long_add(wb->avg_write_bandwidth,  in wb_io_lists_populated()
93 				&wb->bdi->tot_write_bandwidth);  in wb_io_lists_populated()
98 static void wb_io_lists_depopulated(struct bdi_writeback *wb)  in wb_io_lists_depopulated()  argument
100 	if (wb_has_dirty_io(wb) && list_empty(&wb->b_dirty) &&  in wb_io_lists_depopulated()
101 	    list_empty(&wb->b_io) && list_empty(&wb->b_more_io)) {  in wb_io_lists_depopulated()
102 		clear_bit(WB_has_dirty_io, &wb->state);  in wb_io_lists_depopulated()
103 		WARN_ON_ONCE(atomic_long_sub_return(wb->avg_write_bandwidth,  in wb_io_lists_depopulated()
104 					&wb->bdi->tot_write_bandwidth) < 0);  in wb_io_lists_depopulated()
119 				      struct bdi_writeback *wb,  in inode_io_list_move_locked()  argument
122 	assert_spin_locked(&wb->list_lock);  in inode_io_list_move_locked()
127 	if (head != &wb->b_dirty_time)  in inode_io_list_move_locked()
128 		return wb_io_lists_populated(wb);  in inode_io_list_move_locked()
130 	wb_io_lists_depopulated(wb);  in inode_io_list_move_locked()
134 static void wb_wakeup(struct bdi_writeback *wb)  in wb_wakeup()  argument
136 	spin_lock_bh(&wb->work_lock);  in wb_wakeup()
137 	if (test_bit(WB_registered, &wb->state))  in wb_wakeup()
138 		mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_wakeup()
139 	spin_unlock_bh(&wb->work_lock);  in wb_wakeup()
142 static void finish_writeback_work(struct bdi_writeback *wb,  in finish_writeback_work()  argument
158 static void wb_queue_work(struct bdi_writeback *wb,  in wb_queue_work()  argument
161 	trace_writeback_queue(wb, work);  in wb_queue_work()
166 	spin_lock_bh(&wb->work_lock);  in wb_queue_work()
168 	if (test_bit(WB_registered, &wb->state)) {  in wb_queue_work()
169 		list_add_tail(&work->list, &wb->work_list);  in wb_queue_work()
170 		mod_delayed_work(bdi_wq, &wb->dwork, 0);  in wb_queue_work()
172 		finish_writeback_work(wb, work);  in wb_queue_work()
174 	spin_unlock_bh(&wb->work_lock);  in wb_queue_work()
241 	struct bdi_writeback *wb = NULL;  in __inode_attach_wb()  local
248 			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);  in __inode_attach_wb()
252 			wb = wb_get_create(bdi, memcg_css, GFP_ATOMIC);  in __inode_attach_wb()
257 	if (!wb)  in __inode_attach_wb()
258 		wb = &bdi->wb;  in __inode_attach_wb()
264 	if (unlikely(cmpxchg(&inode->i_wb, NULL, wb)))  in __inode_attach_wb()
265 		wb_put(wb);  in __inode_attach_wb()
278 					struct bdi_writeback *wb)  in inode_cgwb_move_to_attached()  argument
280 	assert_spin_locked(&wb->list_lock);  in inode_cgwb_move_to_attached()
284 	if (wb != &wb->bdi->wb)  in inode_cgwb_move_to_attached()
285 		list_move(&inode->i_io_list, &wb->b_attached);  in inode_cgwb_move_to_attached()
288 	wb_io_lists_depopulated(wb);  in inode_cgwb_move_to_attached()
302 	__acquires(&wb->list_lock)  in locked_inode_to_wb_and_lock_list()
305 		struct bdi_writeback *wb = inode_to_wb(inode);  in locked_inode_to_wb_and_lock_list()  local
313 		wb_get(wb);  in locked_inode_to_wb_and_lock_list()
315 		spin_lock(&wb->list_lock);  in locked_inode_to_wb_and_lock_list()
318 		if (likely(wb == inode->i_wb)) {  in locked_inode_to_wb_and_lock_list()
319 			wb_put(wb);	/* @inode already has ref */  in locked_inode_to_wb_and_lock_list()
320 			return wb;  in locked_inode_to_wb_and_lock_list()
323 		spin_unlock(&wb->list_lock);  in locked_inode_to_wb_and_lock_list()
324 		wb_put(wb);  in locked_inode_to_wb_and_lock_list()
338 	__acquires(&wb->list_lock)  in inode_to_wb_and_lock_list()
619 bool cleanup_offline_cgwb(struct bdi_writeback *wb)  in cleanup_offline_cgwb()  argument
634 	for (memcg_css = wb->memcg_css->parent; memcg_css;  in cleanup_offline_cgwb()
636 		isw->new_wb = wb_get_create(wb->bdi, memcg_css, GFP_KERNEL);  in cleanup_offline_cgwb()
641 		isw->new_wb = &wb->bdi->wb; /* wb_get() is noop for bdi's wb */  in cleanup_offline_cgwb()
644 	spin_lock(&wb->list_lock);  in cleanup_offline_cgwb()
645 	list_for_each_entry(inode, &wb->b_attached, i_io_list) {  in cleanup_offline_cgwb()
656 	spin_unlock(&wb->list_lock);  in cleanup_offline_cgwb()
696 	wbc->wb = inode_to_wb(inode);  in wbc_attach_and_unlock_inode()
699 	wbc->wb_id = wbc->wb->memcg_css->id;  in wbc_attach_and_unlock_inode()
706 	wb_get(wbc->wb);  in wbc_attach_and_unlock_inode()
716 	if (unlikely(wb_dying(wbc->wb) && !css_is_dying(wbc->wb->memcg_css)))  in wbc_attach_and_unlock_inode()
760 	struct bdi_writeback *wb = wbc->wb;  in wbc_detach_inode()  local
766 	if (!wb)  in wbc_detach_inode()
793 				wb->avg_write_bandwidth);  in wbc_detach_inode()
839 	wb_put(wbc->wb);  in wbc_detach_inode()
840 	wbc->wb = NULL;  in wbc_detach_inode()
866 	if (!wbc->wb || wbc->no_cgroup_owner)  in wbc_account_cgroup_owner()
917 		struct bdi_writeback *wb;  in inode_congested()  local
921 		wb = unlocked_inode_to_wb_begin(inode, &lock_cookie);  in inode_congested()
922 		congested = wb_congested(wb, cong_bits);  in inode_congested()
927 	return wb_congested(&inode_to_bdi(inode)->wb, cong_bits);  in inode_congested()
940 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)  in wb_split_bdi_pages()  argument
942 	unsigned long this_bw = wb->avg_write_bandwidth;  in wb_split_bdi_pages()
943 	unsigned long tot_bw = atomic_long_read(&wb->bdi->tot_write_bandwidth);  in wb_split_bdi_pages()
975 	struct bdi_writeback *wb = list_entry(&bdi->wb_list,  in bdi_split_work_to_wbs()  local
981 	list_for_each_entry_continue_rcu(wb, &bdi->wb_list, bdi_node) {  in bdi_split_work_to_wbs()
993 		if (!wb_has_dirty_io(wb) &&  in bdi_split_work_to_wbs()
995 		     list_empty(&wb->b_dirty_time)))  in bdi_split_work_to_wbs()
997 		if (skip_if_busy && writeback_in_progress(wb))  in bdi_split_work_to_wbs()
1000 		nr_pages = wb_split_bdi_pages(wb, base_work->nr_pages);  in bdi_split_work_to_wbs()
1007 			wb_queue_work(wb, work);  in bdi_split_work_to_wbs()
1018 		wb_queue_work(wb, work);  in bdi_split_work_to_wbs()
1025 		wb_get(wb);  in bdi_split_work_to_wbs()
1026 		last_wb = wb;  in bdi_split_work_to_wbs()
1053 	struct bdi_writeback *wb;  in cgroup_writeback_by_id()  local
1077 	wb = wb_get_lookup(bdi, memcg_css);  in cgroup_writeback_by_id()
1078 	if (!wb) {  in cgroup_writeback_by_id()
1105 		wb_queue_work(wb, work);  in cgroup_writeback_by_id()
1111 	wb_put(wb);  in cgroup_writeback_by_id()
1162 					struct bdi_writeback *wb)  in inode_cgwb_move_to_attached()  argument
1164 	assert_spin_locked(&wb->list_lock);  in inode_cgwb_move_to_attached()
1169 	wb_io_lists_depopulated(wb);  in inode_cgwb_move_to_attached()
1175 	__acquires(&wb->list_lock)  in locked_inode_to_wb_and_lock_list()
1177 	struct bdi_writeback *wb = inode_to_wb(inode);  in locked_inode_to_wb_and_lock_list()  local
1180 	spin_lock(&wb->list_lock);  in locked_inode_to_wb_and_lock_list()
1181 	return wb;  in locked_inode_to_wb_and_lock_list()
1185 	__acquires(&wb->list_lock)  in inode_to_wb_and_lock_list()
1187 	struct bdi_writeback *wb = inode_to_wb(inode);  in inode_to_wb_and_lock_list()  local
1189 	spin_lock(&wb->list_lock);  in inode_to_wb_and_lock_list()
1190 	return wb;  in inode_to_wb_and_lock_list()
1193 static long wb_split_bdi_pages(struct bdi_writeback *wb, long nr_pages)  in wb_split_bdi_pages()  argument
1204 	if (!skip_if_busy || !writeback_in_progress(&bdi->wb)) {  in bdi_split_work_to_wbs()
1206 		wb_queue_work(&bdi->wb, base_work);  in bdi_split_work_to_wbs()
1222 static void wb_start_writeback(struct bdi_writeback *wb, enum wb_reason reason)  in wb_start_writeback()  argument
1224 	if (!wb_has_dirty_io(wb))  in wb_start_writeback()
1235 	if (test_bit(WB_start_all, &wb->state) ||  in wb_start_writeback()
1236 	    test_and_set_bit(WB_start_all, &wb->state))  in wb_start_writeback()
1239 	wb->start_all_reason = reason;  in wb_start_writeback()
1240 	wb_wakeup(wb);  in wb_start_writeback()
1253 void wb_start_background_writeback(struct bdi_writeback *wb)  in wb_start_background_writeback()  argument
1259 	trace_writeback_wake_background(wb);  in wb_start_background_writeback()
1260 	wb_wakeup(wb);  in wb_start_background_writeback()
1268 	struct bdi_writeback *wb;  in inode_io_list_del()  local
1270 	wb = inode_to_wb_and_lock_list(inode);  in inode_io_list_del()
1275 	wb_io_lists_depopulated(wb);  in inode_io_list_del()
1278 	spin_unlock(&wb->list_lock);  in inode_io_list_del()
1327 static void redirty_tail_locked(struct inode *inode, struct bdi_writeback *wb)  in redirty_tail_locked()  argument
1331 	if (!list_empty(&wb->b_dirty)) {  in redirty_tail_locked()
1334 		tail = wb_inode(wb->b_dirty.next);  in redirty_tail_locked()
1338 	inode_io_list_move_locked(inode, wb, &wb->b_dirty);  in redirty_tail_locked()
1342 static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)  in redirty_tail()  argument
1345 	redirty_tail_locked(inode, wb);  in redirty_tail()
1352 static void requeue_io(struct inode *inode, struct bdi_writeback *wb)  in requeue_io()  argument
1354 	inode_io_list_move_locked(inode, wb, &wb->b_more_io);  in requeue_io()
1445 static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work,  in queue_io()  argument
1451 	assert_spin_locked(&wb->list_lock);  in queue_io()
1452 	list_splice_init(&wb->b_more_io, &wb->b_io);  in queue_io()
1453 	moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, dirtied_before);  in queue_io()
1456 	moved += move_expired_inodes(&wb->b_dirty_time, &wb->b_io,  in queue_io()
1459 		wb_io_lists_populated(wb);  in queue_io()
1460 	trace_writeback_queue_io(wb, work, dirtied_before, moved);  in queue_io()
1534 static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,  in requeue_inode()  argument
1554 		redirty_tail_locked(inode, wb);  in requeue_inode()
1565 			requeue_io(inode, wb);  in requeue_inode()
1574 			redirty_tail_locked(inode, wb);  in requeue_inode()
1582 		redirty_tail_locked(inode, wb);  in requeue_inode()
1585 		inode_io_list_move_locked(inode, wb, &wb->b_dirty_time);  in requeue_inode()
1589 		inode_cgwb_move_to_attached(inode, wb);  in requeue_inode()
1694 	struct bdi_writeback *wb;  in writeback_single_inode()  local
1733 	wb = inode_to_wb_and_lock_list(inode);  in writeback_single_inode()
1741 		inode_cgwb_move_to_attached(inode, wb);  in writeback_single_inode()
1742 	spin_unlock(&wb->list_lock);  in writeback_single_inode()
1749 static long writeback_chunk_size(struct bdi_writeback *wb,  in writeback_chunk_size()  argument
1770 		pages = min(wb->avg_write_bandwidth / 2,  in writeback_chunk_size()
1790 				struct bdi_writeback *wb,  in writeback_sb_inodes()  argument
1807 	while (!list_empty(&wb->b_io)) {  in writeback_sb_inodes()
1808 		struct inode *inode = wb_inode(wb->b_io.prev);  in writeback_sb_inodes()
1818 				redirty_tail(inode, wb);  in writeback_sb_inodes()
1837 			redirty_tail_locked(inode, wb);  in writeback_sb_inodes()
1852 			requeue_io(inode, wb);  in writeback_sb_inodes()
1856 		spin_unlock(&wb->list_lock);  in writeback_sb_inodes()
1867 			spin_lock(&wb->list_lock);  in writeback_sb_inodes()
1873 		write_chunk = writeback_chunk_size(wb, work);  in writeback_sb_inodes()
1912 		if (unlikely(tmp_wb != wb)) {  in writeback_sb_inodes()
1914 			spin_lock(&wb->list_lock);  in writeback_sb_inodes()
1931 static long __writeback_inodes_wb(struct bdi_writeback *wb,  in __writeback_inodes_wb()  argument
1937 	while (!list_empty(&wb->b_io)) {  in __writeback_inodes_wb()
1938 		struct inode *inode = wb_inode(wb->b_io.prev);  in __writeback_inodes_wb()
1947 			redirty_tail(inode, wb);  in __writeback_inodes_wb()
1950 		wrote += writeback_sb_inodes(sb, wb, work);  in __writeback_inodes_wb()
1965 static long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,  in writeback_inodes_wb()  argument
1977 	spin_lock(&wb->list_lock);  in writeback_inodes_wb()
1978 	if (list_empty(&wb->b_io))  in writeback_inodes_wb()
1979 		queue_io(wb, &work, jiffies);  in writeback_inodes_wb()
1980 	__writeback_inodes_wb(wb, &work);  in writeback_inodes_wb()
1981 	spin_unlock(&wb->list_lock);  in writeback_inodes_wb()
2002 static long wb_writeback(struct bdi_writeback *wb,  in wb_writeback()  argument
2012 	spin_lock(&wb->list_lock);  in wb_writeback()
2027 		    !list_empty(&wb->work_list))  in wb_writeback()
2034 		if (work->for_background && !wb_over_bg_thresh(wb))  in wb_writeback()
2049 		trace_writeback_start(wb, work);  in wb_writeback()
2050 		if (list_empty(&wb->b_io))  in wb_writeback()
2051 			queue_io(wb, work, dirtied_before);  in wb_writeback()
2053 			progress = writeback_sb_inodes(work->sb, wb, work);  in wb_writeback()
2055 			progress = __writeback_inodes_wb(wb, work);  in wb_writeback()
2056 		trace_writeback_written(wb, work);  in wb_writeback()
2071 		if (list_empty(&wb->b_more_io))  in wb_writeback()
2078 		trace_writeback_wait(wb, work);  in wb_writeback()
2079 		inode = wb_inode(wb->b_more_io.prev);  in wb_writeback()
2081 		spin_unlock(&wb->list_lock);  in wb_writeback()
2084 		spin_lock(&wb->list_lock);  in wb_writeback()
2086 	spin_unlock(&wb->list_lock);  in wb_writeback()
2095 static struct wb_writeback_work *get_next_work_item(struct bdi_writeback *wb)  in get_next_work_item()  argument
2099 	spin_lock_bh(&wb->work_lock);  in get_next_work_item()
2100 	if (!list_empty(&wb->work_list)) {  in get_next_work_item()
2101 		work = list_entry(wb->work_list.next,  in get_next_work_item()
2105 	spin_unlock_bh(&wb->work_lock);  in get_next_work_item()
2109 static long wb_check_background_flush(struct bdi_writeback *wb)  in wb_check_background_flush()  argument
2111 	if (wb_over_bg_thresh(wb)) {  in wb_check_background_flush()
2121 		return wb_writeback(wb, &work);  in wb_check_background_flush()
2127 static long wb_check_old_data_flush(struct bdi_writeback *wb)  in wb_check_old_data_flush()  argument
2138 	expired = wb->last_old_flush +  in wb_check_old_data_flush()
2143 	wb->last_old_flush = jiffies;  in wb_check_old_data_flush()
2155 		return wb_writeback(wb, &work);  in wb_check_old_data_flush()
2161 static long wb_check_start_all(struct bdi_writeback *wb)  in wb_check_start_all()  argument
2165 	if (!test_bit(WB_start_all, &wb->state))  in wb_check_start_all()
2171 			.nr_pages	= wb_split_bdi_pages(wb, nr_pages),  in wb_check_start_all()
2174 			.reason		= wb->start_all_reason,  in wb_check_start_all()
2177 		nr_pages = wb_writeback(wb, &work);  in wb_check_start_all()
2180 	clear_bit(WB_start_all, &wb->state);  in wb_check_start_all()
2188 static long wb_do_writeback(struct bdi_writeback *wb)  in wb_do_writeback()  argument
2193 	set_bit(WB_writeback_running, &wb->state);  in wb_do_writeback()
2194 	while ((work = get_next_work_item(wb)) != NULL) {  in wb_do_writeback()
2195 		trace_writeback_exec(wb, work);  in wb_do_writeback()
2196 		wrote += wb_writeback(wb, work);  in wb_do_writeback()
2197 		finish_writeback_work(wb, work);  in wb_do_writeback()
2203 	wrote += wb_check_start_all(wb);  in wb_do_writeback()
2208 	wrote += wb_check_old_data_flush(wb);  in wb_do_writeback()
2209 	wrote += wb_check_background_flush(wb);  in wb_do_writeback()
2210 	clear_bit(WB_writeback_running, &wb->state);  in wb_do_writeback()
2221 	struct bdi_writeback *wb = container_of(to_delayed_work(work),  in wb_workfn()  local
2225 	set_worker_desc("flush-%s", bdi_dev_name(wb->bdi));  in wb_workfn()
2229 		   !test_bit(WB_registered, &wb->state))) {  in wb_workfn()
2237 			pages_written = wb_do_writeback(wb);  in wb_workfn()
2239 		} while (!list_empty(&wb->work_list));  in wb_workfn()
2246 		pages_written = writeback_inodes_wb(wb, 1024,  in wb_workfn()
2251 	if (!list_empty(&wb->work_list))  in wb_workfn()
2252 		wb_wakeup(wb);  in wb_workfn()
2253 	else if (wb_has_dirty_io(wb) && dirty_writeback_interval)  in wb_workfn()
2254 		wb_wakeup_delayed(wb);  in wb_workfn()
2266 	struct bdi_writeback *wb;  in __wakeup_flusher_threads_bdi()  local
2271 	list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)  in __wakeup_flusher_threads_bdi()
2272 		wb_start_writeback(wb, reason);  in __wakeup_flusher_threads_bdi()
2326 		struct bdi_writeback *wb;  in wakeup_dirtytime_writeback()  local
2328 		list_for_each_entry_rcu(wb, &bdi->wb_list, bdi_node)  in wakeup_dirtytime_writeback()
2329 			if (!list_empty(&wb->b_dirty_time))  in wakeup_dirtytime_writeback()
2330 				wb_wakeup(wb);  in wakeup_dirtytime_writeback()
2461 			struct bdi_writeback *wb;  in __mark_inode_dirty()  local
2465 			wb = locked_inode_to_wb_and_lock_list(inode);  in __mark_inode_dirty()
2472 				dirty_list = &wb->b_dirty;  in __mark_inode_dirty()
2474 				dirty_list = &wb->b_dirty_time;  in __mark_inode_dirty()
2476 			wakeup_bdi = inode_io_list_move_locked(inode, wb,  in __mark_inode_dirty()
2479 			spin_unlock(&wb->list_lock);  in __mark_inode_dirty()
2489 			    (wb->bdi->capabilities & BDI_CAP_WRITEBACK))  in __mark_inode_dirty()
2490 				wb_wakeup_delayed(wb);  in __mark_inode_dirty()