84 		struct folio *folio = fbatch->folios[i];  in truncate_folio_batch_exceptionals()  local
87 		if (!xa_is_value(folio)) {  in truncate_folio_batch_exceptionals()
88 			fbatch->folios[j++] = folio;  in truncate_folio_batch_exceptionals()
97 		__clear_shadow_entry(mapping, index, folio);  in truncate_folio_batch_exceptionals()
154 void folio_invalidate(struct folio *folio, size_t offset, size_t length)  in folio_invalidate()  argument
156 	const struct address_space_operations *aops = folio->mapping->a_ops;  in folio_invalidate()
159 		aops->invalidate_folio(folio, offset, length);  in folio_invalidate()
173 static void truncate_cleanup_folio(struct folio *folio)  in truncate_cleanup_folio()  argument
175 	if (folio_mapped(folio))  in truncate_cleanup_folio()
176 		unmap_mapping_folio(folio);  in truncate_cleanup_folio()
178 	if (folio_has_private(folio))  in truncate_cleanup_folio()
179 		folio_invalidate(folio, 0, folio_size(folio));  in truncate_cleanup_folio()
186 	folio_cancel_dirty(folio);  in truncate_cleanup_folio()
187 	folio_clear_mappedtodisk(folio);  in truncate_cleanup_folio()
190 int truncate_inode_folio(struct address_space *mapping, struct folio *folio)  in truncate_inode_folio()  argument
192 	if (folio->mapping != mapping)  in truncate_inode_folio()
195 	truncate_cleanup_folio(folio);  in truncate_inode_folio()
196 	filemap_remove_folio(folio);  in truncate_inode_folio()
211 bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)  in truncate_inode_partial_folio()  argument
213 	loff_t pos = folio_pos(folio);  in truncate_inode_partial_folio()
220 	length = folio_size(folio);  in truncate_inode_partial_folio()
226 	folio_wait_writeback(folio);  in truncate_inode_partial_folio()
227 	if (length == folio_size(folio)) {  in truncate_inode_partial_folio()
228 		truncate_inode_folio(folio->mapping, folio);  in truncate_inode_partial_folio()
237 	folio_zero_range(folio, offset, length);  in truncate_inode_partial_folio()
239 	if (folio_has_private(folio))  in truncate_inode_partial_folio()
240 		folio_invalidate(folio, offset, length);  in truncate_inode_partial_folio()
241 	if (!folio_test_large(folio))  in truncate_inode_partial_folio()
243 	if (split_folio(folio) == 0)  in truncate_inode_partial_folio()
245 	if (folio_test_dirty(folio))  in truncate_inode_partial_folio()
247 	truncate_inode_folio(folio->mapping, folio);  in truncate_inode_partial_folio()
271 		struct folio *folio)  in mapping_evict_folio()  argument
273 	if (folio_test_dirty(folio) || folio_test_writeback(folio))  in mapping_evict_folio()
276 	if (folio_ref_count(folio) >  in mapping_evict_folio()
277 			folio_nr_pages(folio) + folio_has_private(folio) + 1)  in mapping_evict_folio()
279 	if (folio_has_private(folio) && !filemap_release_folio(folio, 0))  in mapping_evict_folio()
282 	return remove_mapping(mapping, folio);  in mapping_evict_folio()
297 	struct folio *folio = page_folio(page);  in invalidate_inode_page()  local
298 	struct address_space *mapping = folio_mapping(folio);  in invalidate_inode_page()
303 	return mapping_evict_folio(mapping, folio);  in invalidate_inode_page()
339 	struct folio	*folio;  in truncate_inode_pages_range()  local
378 	folio = __filemap_get_folio(mapping, lstart >> PAGE_SHIFT, FGP_LOCK, 0);  in truncate_inode_pages_range()
379 	if (folio) {  in truncate_inode_pages_range()
380 		same_folio = lend < folio_pos(folio) + folio_size(folio);  in truncate_inode_pages_range()
381 		if (!truncate_inode_partial_folio(folio, lstart, lend)) {  in truncate_inode_pages_range()
382 			start = folio->index + folio_nr_pages(folio);  in truncate_inode_pages_range()
384 				end = folio->index;  in truncate_inode_pages_range()
386 		folio_unlock(folio);  in truncate_inode_pages_range()
387 		folio_put(folio);  in truncate_inode_pages_range()
388 		folio = NULL;  in truncate_inode_pages_range()
392 		folio = __filemap_get_folio(mapping, lend >> PAGE_SHIFT,  in truncate_inode_pages_range()
394 	if (folio) {  in truncate_inode_pages_range()
395 		if (!truncate_inode_partial_folio(folio, lstart, lend))  in truncate_inode_pages_range()
396 			end = folio->index;  in truncate_inode_pages_range()
397 		folio_unlock(folio);  in truncate_inode_pages_range()
398 		folio_put(folio);  in truncate_inode_pages_range()
415 			struct folio *folio = fbatch.folios[i];  in truncate_inode_pages_range()  local
420 			if (xa_is_value(folio))  in truncate_inode_pages_range()
423 			folio_lock(folio);  in truncate_inode_pages_range()
424 			VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);  in truncate_inode_pages_range()
425 			folio_wait_writeback(folio);  in truncate_inode_pages_range()
426 			truncate_inode_folio(mapping, folio);  in truncate_inode_pages_range()
427 			folio_unlock(folio);  in truncate_inode_pages_range()
428 			index = folio_index(folio) + folio_nr_pages(folio) - 1;  in truncate_inode_pages_range()
515 			struct folio *folio = fbatch.folios[i];  in invalidate_mapping_pagevec()  local
520 			if (xa_is_value(folio)) {  in invalidate_mapping_pagevec()
523 								      folio);  in invalidate_mapping_pagevec()
526 			index += folio_nr_pages(folio) - 1;  in invalidate_mapping_pagevec()
528 			ret = mapping_evict_folio(mapping, folio);  in invalidate_mapping_pagevec()
529 			folio_unlock(folio);  in invalidate_mapping_pagevec()
535 				deactivate_file_folio(folio);  in invalidate_mapping_pagevec()
579 					struct folio *folio)  in invalidate_complete_folio2()  argument
581 	if (folio->mapping != mapping)  in invalidate_complete_folio2()
584 	if (folio_has_private(folio) &&  in invalidate_complete_folio2()
585 	    !filemap_release_folio(folio, GFP_KERNEL))  in invalidate_complete_folio2()
590 	if (folio_test_dirty(folio))  in invalidate_complete_folio2()
593 	BUG_ON(folio_has_private(folio));  in invalidate_complete_folio2()
594 	__filemap_remove_folio(folio, NULL);  in invalidate_complete_folio2()
600 	filemap_free_folio(mapping, folio);  in invalidate_complete_folio2()
608 static int folio_launder(struct address_space *mapping, struct folio *folio)  in folio_launder()  argument
610 	if (!folio_test_dirty(folio))  in folio_launder()
612 	if (folio->mapping != mapping || mapping->a_ops->launder_folio == NULL)  in folio_launder()
614 	return mapping->a_ops->launder_folio(folio);  in folio_launder()
646 			struct folio *folio = fbatch.folios[i];  in invalidate_inode_pages2_range()  local
651 			if (xa_is_value(folio)) {  in invalidate_inode_pages2_range()
653 						index, folio))  in invalidate_inode_pages2_range()
658 			if (!did_range_unmap && folio_mapped(folio)) {  in invalidate_inode_pages2_range()
668 			folio_lock(folio);  in invalidate_inode_pages2_range()
669 			VM_BUG_ON_FOLIO(!folio_contains(folio, index), folio);  in invalidate_inode_pages2_range()
670 			if (folio->mapping != mapping) {  in invalidate_inode_pages2_range()
671 				folio_unlock(folio);  in invalidate_inode_pages2_range()
674 			folio_wait_writeback(folio);  in invalidate_inode_pages2_range()
676 			if (folio_mapped(folio))  in invalidate_inode_pages2_range()
677 				unmap_mapping_folio(folio);  in invalidate_inode_pages2_range()
678 			BUG_ON(folio_mapped(folio));  in invalidate_inode_pages2_range()
680 			ret2 = folio_launder(mapping, folio);  in invalidate_inode_pages2_range()
682 				if (!invalidate_complete_folio2(mapping, folio))  in invalidate_inode_pages2_range()
687 			folio_unlock(folio);  in invalidate_inode_pages2_range()