141 static void do_spring_cleaning(struct ce_array *ca)  in do_spring_cleaning()  argument
145 	for (i = 0; i < ca->n; i++) {  in do_spring_cleaning()
146 		u8 decay = DECAY(ca->array[i]);  in do_spring_cleaning()
153 		ca->array[i] &= ~(DECAY_MASK << COUNT_BITS);  in do_spring_cleaning()
154 		ca->array[i] |= (decay << COUNT_BITS);  in do_spring_cleaning()
156 	ca->decay_count = 0;  in do_spring_cleaning()
157 	ca->decays_done++;  in do_spring_cleaning()
184 static int __find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in __find_elem()  argument
187 	int min = 0, max = ca->n;  in __find_elem()
192 		this_pfn = PFN(ca->array[tmp]);  in __find_elem()
207 	this_pfn = PFN(ca->array[min]);  in __find_elem()
215 static int find_elem(struct ce_array *ca, u64 pfn, unsigned int *to)  in find_elem()  argument
219 	if (!ca->n) {  in find_elem()
223 	return __find_elem(ca, pfn, to);  in find_elem()
226 static void del_elem(struct ce_array *ca, int idx)  in del_elem()  argument
229 	if (ca->n - (idx + 1))  in del_elem()
230 		memmove((void *)&ca->array[idx],  in del_elem()
231 			(void *)&ca->array[idx + 1],  in del_elem()
232 			(ca->n - (idx + 1)) * sizeof(u64));  in del_elem()
234 	ca->n--;  in del_elem()
237 static u64 del_lru_elem_unlocked(struct ce_array *ca)  in del_lru_elem_unlocked()  argument
242 	for (i = 0; i < ca->n; i++) {  in del_lru_elem_unlocked()
243 		unsigned int this = FULL_COUNT(ca->array[i]);  in del_lru_elem_unlocked()
251 	del_elem(ca, min_idx);  in del_lru_elem_unlocked()
253 	return PFN(ca->array[min_idx]);  in del_lru_elem_unlocked()
262 	struct ce_array *ca = &ce_arr;  in del_lru_elem()  local
265 	if (!ca->n)  in del_lru_elem()
269 	pfn = del_lru_elem_unlocked(ca);  in del_lru_elem()
278 	struct ce_array *ca = &ce_arr;  in cec_add_elem()  local
289 	ca->ces_entered++;  in cec_add_elem()
293 	if (ca->n == MAX_ELEMS)  in cec_add_elem()
294 		WARN_ON(!del_lru_elem_unlocked(ca));  in cec_add_elem()
296 	ret = find_elem(ca, pfn, &to);  in cec_add_elem()
301 		memmove((void *)&ca->array[to + 1],  in cec_add_elem()
302 			(void *)&ca->array[to],  in cec_add_elem()
303 			(ca->n - to) * sizeof(u64));  in cec_add_elem()
305 		ca->array[to] = (pfn << PAGE_SHIFT) |  in cec_add_elem()
308 		ca->n++;  in cec_add_elem()
315 	count = COUNT(ca->array[to]);  in cec_add_elem()
318 		ca->array[to] |= (DECAY_MASK << COUNT_BITS);  in cec_add_elem()
319 		ca->array[to]++;  in cec_add_elem()
323 		u64 pfn = ca->array[to] >> PAGE_SHIFT;  in cec_add_elem()
331 			ca->pfns_poisoned++;  in cec_add_elem()
334 		del_elem(ca, to);  in cec_add_elem()
346 	ca->decay_count++;  in cec_add_elem()
348 	if (ca->decay_count >= CLEAN_ELEMS)  in cec_add_elem()
349 		do_spring_cleaning(ca);  in cec_add_elem()
405 	struct ce_array *ca = &ce_arr;  in array_dump()  local
411 	seq_printf(m, "{ n: %d\n", ca->n);  in array_dump()
412 	for (i = 0; i < ca->n; i++) {  in array_dump()
413 		u64 this = PFN(ca->array[i]);  in array_dump()
415 		seq_printf(m, " %03d: [%016llx|%03llx]\n", i, this, FULL_COUNT(ca->array[i]));  in array_dump()
425 		   ca->ces_entered, ca->pfns_poisoned);  in array_dump()
427 	seq_printf(m, "Flags: 0x%x\n", ca->flags);  in array_dump()
430 	seq_printf(m, "Decays: %lld\n", ca->decays_done);  in array_dump()