77 void num_poisoned_pages_inc(unsigned long pfn)  in num_poisoned_pages_inc()  argument
80 	memblk_nr_poison_inc(pfn);  in num_poisoned_pages_inc()
83 void num_poisoned_pages_sub(unsigned long pfn, long i)  in num_poisoned_pages_sub()  argument
86 	if (pfn != -1UL)  in num_poisoned_pages_sub()
87 		memblk_nr_poison_sub(pfn, i);  in num_poisoned_pages_sub()
328 static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags)  in kill_proc()  argument
335 			pfn, t->comm, t->pid);  in kill_proc()
511 		unsigned long pfn, int flags)  in kill_procs()  argument
524 				       pfn, tk->tsk->comm, tk->tsk->pid);  in kill_procs()
535 			else if (kill_proc(tk, pfn, flags) < 0)  in kill_procs()
537 				       pfn, tk->tsk->comm, tk->tsk->pid);  in kill_procs()
722 	unsigned long pfn;  member
735 	unsigned long pfn = 0;  in check_hwpoisoned_entry()  local
738 		pfn = pte_pfn(pte);  in check_hwpoisoned_entry()
743 			pfn = swp_offset_pfn(swp);  in check_hwpoisoned_entry()
746 	if (!pfn || pfn != poisoned_pfn)  in check_hwpoisoned_entry()
758 	unsigned long pfn;  in check_hwpoisoned_pmd_entry()  local
763 	pfn = pmd_pfn(pmd);  in check_hwpoisoned_pmd_entry()
764 	if (pfn <= hwp->pfn && hwp->pfn < pfn + HPAGE_PMD_NR) {  in check_hwpoisoned_pmd_entry()
765 		hwpoison_vaddr = addr + ((hwp->pfn - pfn) << PAGE_SHIFT);  in check_hwpoisoned_pmd_entry()
801 					     hwp->pfn, &hwp->tk);  in hwpoison_pte_range()
821 				      hwp->pfn, &hwp->tk);  in hwpoison_hugetlb_range()
846 static int kill_accessing_process(struct task_struct *p, unsigned long pfn,  in kill_accessing_process()  argument
851 		.pfn = pfn,  in kill_accessing_process()
862 		kill_proc(&priv.tk, pfn, flags);  in kill_accessing_process()
930 static int truncate_error_page(struct page *p, unsigned long pfn,  in truncate_error_page()  argument
940 			pr_info("%#lx: Failed to punch page: %d\n", pfn, err);  in truncate_error_page()
942 			pr_info("%#lx: failed to release buffers\n", pfn);  in truncate_error_page()
953 			pr_info("%#lx: Failed to invalidate\n",	pfn);  in truncate_error_page()
1283 static void update_per_node_mf_stats(unsigned long pfn,  in update_per_node_mf_stats()  argument
1289 	nid = pfn_to_nid(pfn);  in update_per_node_mf_stats()
1291 		WARN_ONCE(1, "Memory failure: pfn=%#lx, invalid nid=%d", pfn, nid);  in update_per_node_mf_stats()
1320 static int action_result(unsigned long pfn, enum mf_action_page_type type,  in action_result()  argument
1323 	trace_memory_failure_event(pfn, type, result);  in action_result()
1325 	num_poisoned_pages_inc(pfn);  in action_result()
1327 	update_per_node_mf_stats(pfn, result);  in action_result()
1330 		pfn, action_page_types[type], action_name[result]);  in action_result()
1336 			unsigned long pfn)  in page_action()  argument
1348 	return action_result(pfn, ps->type, result);  in page_action()
1550 static bool hwpoison_user_mappings(struct page *p, unsigned long pfn,  in hwpoison_user_mappings()  argument
1578 		pr_err("%#lx: keeping poisoned page in swap cache\n", pfn);  in hwpoison_user_mappings()
1596 				pfn);  in hwpoison_user_mappings()
1620 			pr_info("%#lx: could not lock mapping for mapped huge page\n", pfn);  in hwpoison_user_mappings()
1628 		       pfn, page_mapcount(hpage));  in hwpoison_user_mappings()
1649 	kill_procs(&tokill, forcekill, !unmap_success, pfn, flags);  in hwpoison_user_mappings()
1654 static int identify_page_state(unsigned long pfn, struct page *p,  in identify_page_state()  argument
1674 	return page_action(ps, p, pfn);  in identify_page_state()
1691 static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,  in unmap_and_kill()  argument
1713 	kill_procs(to_kill, flags & MF_MUST_KILL, false, pfn, flags);  in unmap_and_kill()
1716 static int mf_generic_kill_procs(unsigned long long pfn, int flags,  in mf_generic_kill_procs()  argument
1719 	struct page *page = pfn_to_page(pfn);  in mf_generic_kill_procs()
1774 	unmap_and_kill(&to_kill, pfn, page->mapping, page->index, flags);  in mf_generic_kill_procs()
1970 int __get_huge_page_for_hwpoison(unsigned long pfn, int flags,  in __get_huge_page_for_hwpoison()  argument
1973 	struct page *page = pfn_to_page(pfn);  in __get_huge_page_for_hwpoison()
2023 static int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)  in try_memory_failure_hugetlb()  argument
2026 	struct page *p = pfn_to_page(pfn);  in try_memory_failure_hugetlb()
2033 	res = get_huge_page_for_hwpoison(pfn, flags, &migratable_cleared);  in try_memory_failure_hugetlb()
2038 		pr_err("%#lx: already hardware poisoned\n", pfn);  in try_memory_failure_hugetlb()
2049 		return action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);  in try_memory_failure_hugetlb()
2077 		return action_result(pfn, MF_MSG_FREE_HUGE, res);  in try_memory_failure_hugetlb()
2082 	if (!hwpoison_user_mappings(p, pfn, flags, &folio->page)) {  in try_memory_failure_hugetlb()
2084 		return action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);  in try_memory_failure_hugetlb()
2087 	return identify_page_state(pfn, p, page_flags);  in try_memory_failure_hugetlb()
2091 static inline int try_memory_failure_hugetlb(unsigned long pfn, int flags, int *hugetlb)  in try_memory_failure_hugetlb()  argument
2103 static void put_ref_page(unsigned long pfn, int flags)  in put_ref_page()  argument
2110 	page = pfn_to_page(pfn);  in put_ref_page()
2115 static int memory_failure_dev_pagemap(unsigned long pfn, int flags,  in memory_failure_dev_pagemap()  argument
2121 	if (!pgmap_pfn_valid(pgmap, pfn))  in memory_failure_dev_pagemap()
2129 		rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags);  in memory_failure_dev_pagemap()
2138 	rc = mf_generic_kill_procs(pfn, flags, pgmap);  in memory_failure_dev_pagemap()
2143 		action_result(pfn, MF_MSG_DAX, rc ? MF_FAILED : MF_RECOVERED);  in memory_failure_dev_pagemap()
2168 int memory_failure(unsigned long pfn, int flags)  in memory_failure()  argument
2179 		panic("Memory failure on page %lx", pfn);  in memory_failure()
2186 	p = pfn_to_online_page(pfn);  in memory_failure()
2188 		res = arch_memory_failure(pfn, flags);  in memory_failure()
2192 		if (pfn_valid(pfn)) {  in memory_failure()
2193 			pgmap = get_dev_pagemap(pfn, NULL);  in memory_failure()
2194 			put_ref_page(pfn, flags);  in memory_failure()
2196 				res = memory_failure_dev_pagemap(pfn, flags,  in memory_failure()
2201 		pr_err("%#lx: memory outside kernel control\n", pfn);  in memory_failure()
2207 	res = try_memory_failure_hugetlb(pfn, flags, &hugetlb);  in memory_failure()
2212 		pr_err("%#lx: already hardware poisoned\n", pfn);  in memory_failure()
2215 			res = kill_accessing_process(current, pfn, flags);  in memory_failure()
2248 				res = action_result(pfn, MF_MSG_BUDDY, res);  in memory_failure()
2250 				res = action_result(pfn, MF_MSG_KERNEL_HIGH_ORDER, MF_IGNORED);  in memory_failure()
2254 			res = action_result(pfn, MF_MSG_UNKNOWN, MF_IGNORED);  in memory_failure()
2276 			res = action_result(pfn, MF_MSG_UNSPLIT_THP, MF_IGNORED);  in memory_failure()
2309 		res = action_result(pfn, MF_MSG_DIFFERENT_COMPOUND, MF_IGNORED);  in memory_failure()
2348 	if (!hwpoison_user_mappings(p, pfn, flags, p)) {  in memory_failure()
2349 		res = action_result(pfn, MF_MSG_UNMAP_FAILED, MF_IGNORED);  in memory_failure()
2357 		res = action_result(pfn, MF_MSG_TRUNCATED_LRU, MF_IGNORED);  in memory_failure()
2362 	res = identify_page_state(pfn, p, page_flags);  in memory_failure()
2377 	unsigned long pfn;  member
2406 void memory_failure_queue(unsigned long pfn, int flags)  in memory_failure_queue()  argument
2411 		.pfn =		pfn,  in memory_failure_queue()
2421 		       pfn);  in memory_failure_queue()
2442 			soft_offline_page(entry.pfn, entry.flags);  in memory_failure_work_func()
2444 			memory_failure(entry.pfn, entry.flags);  in memory_failure_work_func()
2481 #define unpoison_pr_info(fmt, pfn, rs)			\  argument
2484 		pr_info(fmt, pfn);			\
2499 int unpoison_memory(unsigned long pfn)  in unpoison_memory()  argument
2509 	if (!pfn_valid(pfn))  in unpoison_memory()
2512 	p = pfn_to_page(pfn);  in unpoison_memory()
2519 				 pfn, &unpoison_rs);  in unpoison_memory()
2526 				 pfn, &unpoison_rs);  in unpoison_memory()
2532 				 pfn, &unpoison_rs);  in unpoison_memory()
2546 				 pfn, &unpoison_rs);  in unpoison_memory()
2552 				 pfn, &unpoison_rs);  in unpoison_memory()
2571 					 pfn, &unpoison_rs);  in unpoison_memory()
2594 			num_poisoned_pages_sub(pfn, 1);  in unpoison_memory()
2644 	unsigned long pfn = page_to_pfn(page);  in soft_offline_in_use_page()  local
2656 			pr_info("soft offline: %#lx: thp split failed\n", pfn);  in soft_offline_in_use_page()
2668 		pr_info("soft offline: %#lx page already poisoned\n", pfn);  in soft_offline_in_use_page()
2681 		pr_info("soft_offline: %#lx: invalidated\n", pfn);  in soft_offline_in_use_page()
2699 				pfn, msg_page[huge], ret, &page->flags);  in soft_offline_in_use_page()
2705 			pfn, msg_page[huge], page_count(page), &page->flags);  in soft_offline_in_use_page()
2735 int soft_offline_page(unsigned long pfn, int flags)  in soft_offline_page()  argument
2741 	if (!pfn_valid(pfn)) {  in soft_offline_page()
2747 	page = pfn_to_online_page(pfn);  in soft_offline_page()
2749 		put_ref_page(pfn, flags);  in soft_offline_page()
2756 		pr_info("%s: %#lx page already poisoned\n", __func__, pfn);  in soft_offline_page()
2757 		put_ref_page(pfn, flags);  in soft_offline_page()