1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * DAMON Primitives for The Physical Address Space
4 *
5 * Author: SeongJae Park <sj@kernel.org>
6 */
7
8 #define pr_fmt(fmt) "damon-pa: " fmt
9
10 #include <linux/mmu_notifier.h>
11 #include <linux/page_idle.h>
12 #include <linux/pagemap.h>
13 #include <linux/rmap.h>
14 #include <linux/swap.h>
15
16 #include "../internal.h"
17 #include "ops-common.h"
18
__damon_pa_mkold(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)19 static bool __damon_pa_mkold(struct folio *folio, struct vm_area_struct *vma,
20 unsigned long addr, void *arg)
21 {
22 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
23
24 while (page_vma_mapped_walk(&pvmw)) {
25 addr = pvmw.address;
26 if (pvmw.pte)
27 damon_ptep_mkold(pvmw.pte, vma->vm_mm, addr);
28 else
29 damon_pmdp_mkold(pvmw.pmd, vma->vm_mm, addr);
30 }
31 return true;
32 }
33
damon_pa_mkold(unsigned long paddr)34 static void damon_pa_mkold(unsigned long paddr)
35 {
36 struct folio *folio;
37 struct page *page = damon_get_page(PHYS_PFN(paddr));
38 struct rmap_walk_control rwc = {
39 .rmap_one = __damon_pa_mkold,
40 .anon_lock = folio_lock_anon_vma_read,
41 };
42 bool need_lock;
43
44 if (!page)
45 return;
46 folio = page_folio(page);
47
48 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
49 folio_set_idle(folio);
50 goto out;
51 }
52
53 need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
54 if (need_lock && !folio_trylock(folio))
55 goto out;
56
57 rmap_walk(folio, &rwc);
58
59 if (need_lock)
60 folio_unlock(folio);
61
62 out:
63 folio_put(folio);
64 }
65
__damon_pa_prepare_access_check(struct damon_region * r)66 static void __damon_pa_prepare_access_check(struct damon_region *r)
67 {
68 r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
69
70 damon_pa_mkold(r->sampling_addr);
71 }
72
damon_pa_prepare_access_checks(struct damon_ctx * ctx)73 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
74 {
75 struct damon_target *t;
76 struct damon_region *r;
77
78 damon_for_each_target(t, ctx) {
79 damon_for_each_region(r, t)
80 __damon_pa_prepare_access_check(r);
81 }
82 }
83
84 struct damon_pa_access_chk_result {
85 unsigned long page_sz;
86 bool accessed;
87 };
88
__damon_pa_young(struct folio * folio,struct vm_area_struct * vma,unsigned long addr,void * arg)89 static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
90 unsigned long addr, void *arg)
91 {
92 struct damon_pa_access_chk_result *result = arg;
93 DEFINE_FOLIO_VMA_WALK(pvmw, folio, vma, addr, 0);
94
95 result->accessed = false;
96 result->page_sz = PAGE_SIZE;
97 while (page_vma_mapped_walk(&pvmw)) {
98 addr = pvmw.address;
99 if (pvmw.pte) {
100 result->accessed = pte_young(*pvmw.pte) ||
101 !folio_test_idle(folio) ||
102 mmu_notifier_test_young(vma->vm_mm, addr);
103 } else {
104 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
105 result->accessed = pmd_young(*pvmw.pmd) ||
106 !folio_test_idle(folio) ||
107 mmu_notifier_test_young(vma->vm_mm, addr);
108 result->page_sz = HPAGE_PMD_SIZE;
109 #else
110 WARN_ON_ONCE(1);
111 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
112 }
113 if (result->accessed) {
114 page_vma_mapped_walk_done(&pvmw);
115 break;
116 }
117 }
118
119 /* If accessed, stop walking */
120 return !result->accessed;
121 }
122
damon_pa_young(unsigned long paddr,unsigned long * page_sz)123 static bool damon_pa_young(unsigned long paddr, unsigned long *page_sz)
124 {
125 struct folio *folio;
126 struct page *page = damon_get_page(PHYS_PFN(paddr));
127 struct damon_pa_access_chk_result result = {
128 .page_sz = PAGE_SIZE,
129 .accessed = false,
130 };
131 struct rmap_walk_control rwc = {
132 .arg = &result,
133 .rmap_one = __damon_pa_young,
134 .anon_lock = folio_lock_anon_vma_read,
135 };
136 bool need_lock;
137
138 if (!page)
139 return false;
140 folio = page_folio(page);
141
142 if (!folio_mapped(folio) || !folio_raw_mapping(folio)) {
143 if (folio_test_idle(folio))
144 result.accessed = false;
145 else
146 result.accessed = true;
147 folio_put(folio);
148 goto out;
149 }
150
151 need_lock = !folio_test_anon(folio) || folio_test_ksm(folio);
152 if (need_lock && !folio_trylock(folio)) {
153 folio_put(folio);
154 return false;
155 }
156
157 rmap_walk(folio, &rwc);
158
159 if (need_lock)
160 folio_unlock(folio);
161 folio_put(folio);
162
163 out:
164 *page_sz = result.page_sz;
165 return result.accessed;
166 }
167
__damon_pa_check_access(struct damon_region * r)168 static void __damon_pa_check_access(struct damon_region *r)
169 {
170 static unsigned long last_addr;
171 static unsigned long last_page_sz = PAGE_SIZE;
172 static bool last_accessed;
173
174 /* If the region is in the last checked page, reuse the result */
175 if (ALIGN_DOWN(last_addr, last_page_sz) ==
176 ALIGN_DOWN(r->sampling_addr, last_page_sz)) {
177 if (last_accessed)
178 r->nr_accesses++;
179 return;
180 }
181
182 last_accessed = damon_pa_young(r->sampling_addr, &last_page_sz);
183 if (last_accessed)
184 r->nr_accesses++;
185
186 last_addr = r->sampling_addr;
187 }
188
damon_pa_check_accesses(struct damon_ctx * ctx)189 static unsigned int damon_pa_check_accesses(struct damon_ctx *ctx)
190 {
191 struct damon_target *t;
192 struct damon_region *r;
193 unsigned int max_nr_accesses = 0;
194
195 damon_for_each_target(t, ctx) {
196 damon_for_each_region(r, t) {
197 __damon_pa_check_access(r);
198 max_nr_accesses = max(r->nr_accesses, max_nr_accesses);
199 }
200 }
201
202 return max_nr_accesses;
203 }
204
damon_pa_pageout(struct damon_region * r)205 static unsigned long damon_pa_pageout(struct damon_region *r)
206 {
207 unsigned long addr, applied;
208 LIST_HEAD(page_list);
209
210 for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
211 struct page *page = damon_get_page(PHYS_PFN(addr));
212
213 if (!page)
214 continue;
215
216 ClearPageReferenced(page);
217 test_and_clear_page_young(page);
218 if (isolate_lru_page(page)) {
219 put_page(page);
220 continue;
221 }
222 if (PageUnevictable(page)) {
223 putback_lru_page(page);
224 } else {
225 list_add(&page->lru, &page_list);
226 put_page(page);
227 }
228 }
229 applied = reclaim_pages(&page_list);
230 cond_resched();
231 return applied * PAGE_SIZE;
232 }
233
damon_pa_mark_accessed_or_deactivate(struct damon_region * r,bool mark_accessed)234 static inline unsigned long damon_pa_mark_accessed_or_deactivate(
235 struct damon_region *r, bool mark_accessed)
236 {
237 unsigned long addr, applied = 0;
238
239 for (addr = r->ar.start; addr < r->ar.end; addr += PAGE_SIZE) {
240 struct page *page = damon_get_page(PHYS_PFN(addr));
241
242 if (!page)
243 continue;
244 if (mark_accessed)
245 mark_page_accessed(page);
246 else
247 deactivate_page(page);
248 put_page(page);
249 applied++;
250 }
251 return applied * PAGE_SIZE;
252 }
253
damon_pa_mark_accessed(struct damon_region * r)254 static unsigned long damon_pa_mark_accessed(struct damon_region *r)
255 {
256 return damon_pa_mark_accessed_or_deactivate(r, true);
257 }
258
damon_pa_deactivate_pages(struct damon_region * r)259 static unsigned long damon_pa_deactivate_pages(struct damon_region *r)
260 {
261 return damon_pa_mark_accessed_or_deactivate(r, false);
262 }
263
damon_pa_apply_scheme(struct damon_ctx * ctx,struct damon_target * t,struct damon_region * r,struct damos * scheme)264 static unsigned long damon_pa_apply_scheme(struct damon_ctx *ctx,
265 struct damon_target *t, struct damon_region *r,
266 struct damos *scheme)
267 {
268 switch (scheme->action) {
269 case DAMOS_PAGEOUT:
270 return damon_pa_pageout(r);
271 case DAMOS_LRU_PRIO:
272 return damon_pa_mark_accessed(r);
273 case DAMOS_LRU_DEPRIO:
274 return damon_pa_deactivate_pages(r);
275 case DAMOS_STAT:
276 break;
277 default:
278 /* DAMOS actions that not yet supported by 'paddr'. */
279 break;
280 }
281 return 0;
282 }
283
damon_pa_scheme_score(struct damon_ctx * context,struct damon_target * t,struct damon_region * r,struct damos * scheme)284 static int damon_pa_scheme_score(struct damon_ctx *context,
285 struct damon_target *t, struct damon_region *r,
286 struct damos *scheme)
287 {
288 switch (scheme->action) {
289 case DAMOS_PAGEOUT:
290 return damon_cold_score(context, r, scheme);
291 case DAMOS_LRU_PRIO:
292 return damon_hot_score(context, r, scheme);
293 case DAMOS_LRU_DEPRIO:
294 return damon_cold_score(context, r, scheme);
295 default:
296 break;
297 }
298
299 return DAMOS_MAX_SCORE;
300 }
301
damon_pa_initcall(void)302 static int __init damon_pa_initcall(void)
303 {
304 struct damon_operations ops = {
305 .id = DAMON_OPS_PADDR,
306 .init = NULL,
307 .update = NULL,
308 .prepare_access_checks = damon_pa_prepare_access_checks,
309 .check_accesses = damon_pa_check_accesses,
310 .reset_aggregated = NULL,
311 .target_valid = NULL,
312 .cleanup = NULL,
313 .apply_scheme = damon_pa_apply_scheme,
314 .get_scheme_score = damon_pa_scheme_score,
315 };
316
317 return damon_register_ops(&ops);
318 };
319
320 subsys_initcall(damon_pa_initcall);
321
