142 static void do_spring_cleaning(struct ce_array *ca)  in do_spring_cleaning()  argument
146 	for (i = 0; i < ca->n; i++) {  in do_spring_cleaning()
147 		u8 decay = DECAY(ca->array[i]);  in do_spring_cleaning()
154 		ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);  in do_spring_cleaning()
155 		ca->array[i] |= (decay << COUNT_BITS);  in do_spring_cleaning()
157 	ca->decay_count = 0;  in do_spring_cleaning()
158 	ca->decays_done++;  in do_spring_cleaning()
186 static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in __find_elem()  argument
188 	int min = 0, max = ca->n - 1;  in __find_elem()
194 		this_pfn = PFN(ca->array[i]);  in __find_elem()
223 static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in find_elem()  argument
227 	if (!ca->n) {  in find_elem()
231 	return __find_elem(ca, pfn, to);  in find_elem()
234 static void del_elem(struct ce_array *ca, int idx)  in del_elem()  argument
237 	if (ca->n - (idx + 1))  in del_elem()
238 		memmove((void *)&ca->array[idx],  in del_elem()
239 			(void *)&ca->array[idx + 1],  in del_elem()
240 			(ca->n - (idx + 1)) * sizeof(u64));  in del_elem()
242 	ca->n--;  in del_elem()
245 static u64 del_lru_elem_unlocked(struct ce_array *ca)  in del_lru_elem_unlocked()  argument
250 	for (i = 0; i < ca->n; i++) {  in del_lru_elem_unlocked()
251 		unsigned int this = FULL_COUNT(ca->array[i]);  in del_lru_elem_unlocked()
259 	del_elem(ca, min_idx);  in del_lru_elem_unlocked()
261 	return PFN(ca->array[min_idx]);  in del_lru_elem_unlocked()
270 	struct ce_array *ca = &ce_arr;  in del_lru_elem()  local
273 	if (!ca->n)  in del_lru_elem()
277 	pfn = del_lru_elem_unlocked(ca);  in del_lru_elem()
283 static bool sanity_check(struct ce_array *ca)  in sanity_check()  argument
289 	for (i = 0; i < ca->n; i++) {  in sanity_check()
290 		u64 this = PFN(ca->array[i]);  in sanity_check()
301 	pr_info("Sanity check dump:\n{ n: %d\n", ca->n);  in sanity_check()
302 	for (i = 0; i < ca->n; i++) {  in sanity_check()
303 		u64 this = PFN(ca->array[i]);  in sanity_check()
305 		pr_info(" %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));  in sanity_check()
314 	struct ce_array *ca = &ce_arr;  in cec_add_elem()  local
327 	ca->ces_entered++;  in cec_add_elem()
330 	if (ca->n == MAX_ELEMS)  in cec_add_elem()
331 		WARN_ON(!del_lru_elem_unlocked(ca));  in cec_add_elem()
333 	ret = find_elem(ca, pfn, &to);  in cec_add_elem()
338 		memmove((void *)&ca->array[to + 1],  in cec_add_elem()
339 			(void *)&ca->array[to],  in cec_add_elem()
340 			(ca->n - to) * sizeof(u64));  in cec_add_elem()
342 		ca->array[to] = pfn << PAGE_SHIFT;  in cec_add_elem()
343 		ca->n++;  in cec_add_elem()
347 	ca->array[to] |= DECAY_MASK << COUNT_BITS;  in cec_add_elem()
348 	ca->array[to]++;  in cec_add_elem()
351 	count = COUNT(ca->array[to]);  in cec_add_elem()
353 		u64 pfn = ca->array[to] >> PAGE_SHIFT;  in cec_add_elem()
361 			ca->pfns_poisoned++;  in cec_add_elem()
364 		del_elem(ca, to);  in cec_add_elem()
375 	ca->decay_count++;  in cec_add_elem()
377 	if (ca->decay_count >= CLEAN_ELEMS)  in cec_add_elem()
378 		do_spring_cleaning(ca);  in cec_add_elem()
380 	WARN_ON_ONCE(sanity_check(ca));  in cec_add_elem()
440 	struct ce_array *ca = &ce_arr;  in array_dump()  local
445 	seq_printf(m, "{ n: %d\n", ca->n);  in array_dump()
446 	for (i = 0; i < ca->n; i++) {  in array_dump()
447 		u64 this = PFN(ca->array[i]);  in array_dump()
450 			   i, this, bins[DECAY(ca->array[i])], COUNT(ca->array[i]));  in array_dump()
456 		   ca->ces_entered, ca->pfns_poisoned);  in array_dump()
458 	seq_printf(m, "Flags: 0x%x\n", ca->flags);  in array_dump()
461 	seq_printf(m, "Decays: %lld\n", ca->decays_done);  in array_dump()