142 static void __bpf_lru_list_rotate_active(struct bpf_lru *lru,  in __bpf_lru_list_rotate_active()  argument
156 		if (++i == lru->nr_scans || node == first_node)  in __bpf_lru_list_rotate_active()
169 static void __bpf_lru_list_rotate_inactive(struct bpf_lru *lru,  in __bpf_lru_list_rotate_inactive()  argument
185 	while (i < lru->nr_scans) {  in __bpf_lru_list_rotate_inactive()
209 __bpf_lru_list_shrink_inactive(struct bpf_lru *lru,  in __bpf_lru_list_shrink_inactive()  argument
223 		} else if (lru->del_from_htab(lru->del_arg, node)) {  in __bpf_lru_list_shrink_inactive()
230 		if (++i == lru->nr_scans)  in __bpf_lru_list_shrink_inactive()
240 static void __bpf_lru_list_rotate(struct bpf_lru *lru, struct bpf_lru_list *l)  in __bpf_lru_list_rotate()  argument
243 		__bpf_lru_list_rotate_active(lru, l);  in __bpf_lru_list_rotate()
245 	__bpf_lru_list_rotate_inactive(lru, l);  in __bpf_lru_list_rotate()
258 static unsigned int __bpf_lru_list_shrink(struct bpf_lru *lru,  in __bpf_lru_list_shrink()  argument
269 	nshrinked = __bpf_lru_list_shrink_inactive(lru, l, tgt_nshrink,  in __bpf_lru_list_shrink()
282 		if (lru->del_from_htab(lru->del_arg, node)) {  in __bpf_lru_list_shrink()
321 static void bpf_lru_list_pop_free_to_local(struct bpf_lru *lru,  in bpf_lru_list_pop_free_to_local()  argument
324 	struct bpf_lru_list *l = &lru->common_lru.lru_list;  in bpf_lru_list_pop_free_to_local()
332 	__bpf_lru_list_rotate(lru, l);  in bpf_lru_list_pop_free_to_local()
343 		__bpf_lru_list_shrink(lru, l, LOCAL_FREE_TARGET - nfree,  in bpf_lru_list_pop_free_to_local()
350 static void __local_list_add_pending(struct bpf_lru *lru,  in __local_list_add_pending()  argument
356 	*(u32 *)((void *)node + lru->hash_offset) = hash;  in __local_list_add_pending()
378 __local_list_pop_pending(struct bpf_lru *lru, struct bpf_lru_locallist *loc_l)  in __local_list_pop_pending()  argument
388 		    lru->del_from_htab(lru->del_arg, node)) {  in __local_list_pop_pending()
402 static struct bpf_lru_node *bpf_percpu_lru_pop_free(struct bpf_lru *lru,  in bpf_percpu_lru_pop_free()  argument
411 	l = per_cpu_ptr(lru->percpu_lru, cpu);  in bpf_percpu_lru_pop_free()
415 	__bpf_lru_list_rotate(lru, l);  in bpf_percpu_lru_pop_free()
419 		__bpf_lru_list_shrink(lru, l, PERCPU_FREE_TARGET, free_list,  in bpf_percpu_lru_pop_free()
424 		*(u32 *)((void *)node + lru->hash_offset) = hash;  in bpf_percpu_lru_pop_free()
434 static struct bpf_lru_node *bpf_common_lru_pop_free(struct bpf_lru *lru,  in bpf_common_lru_pop_free()  argument
438 	struct bpf_common_lru *clru = &lru->common_lru;  in bpf_common_lru_pop_free()
450 		bpf_lru_list_pop_free_to_local(lru, loc_l);  in bpf_common_lru_pop_free()
455 		__local_list_add_pending(lru, loc_l, cpu, node, hash);  in bpf_common_lru_pop_free()
479 			node = __local_list_pop_pending(lru, steal_loc_l);  in bpf_common_lru_pop_free()
490 		__local_list_add_pending(lru, loc_l, cpu, node, hash);  in bpf_common_lru_pop_free()
497 struct bpf_lru_node *bpf_lru_pop_free(struct bpf_lru *lru, u32 hash)  in bpf_lru_pop_free()  argument
499 	if (lru->percpu)  in bpf_lru_pop_free()
500 		return bpf_percpu_lru_pop_free(lru, hash);  in bpf_lru_pop_free()
502 		return bpf_common_lru_pop_free(lru, hash);  in bpf_lru_pop_free()
505 static void bpf_common_lru_push_free(struct bpf_lru *lru,  in bpf_common_lru_push_free()  argument
517 		loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);  in bpf_common_lru_push_free()
535 	bpf_lru_list_push_free(&lru->common_lru.lru_list, node);  in bpf_common_lru_push_free()
538 static void bpf_percpu_lru_push_free(struct bpf_lru *lru,  in bpf_percpu_lru_push_free()  argument
544 	l = per_cpu_ptr(lru->percpu_lru, node->cpu);  in bpf_percpu_lru_push_free()
553 void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node)  in bpf_lru_push_free()  argument
555 	if (lru->percpu)  in bpf_lru_push_free()
556 		bpf_percpu_lru_push_free(lru, node);  in bpf_lru_push_free()
558 		bpf_common_lru_push_free(lru, node);  in bpf_lru_push_free()
561 static void bpf_common_lru_populate(struct bpf_lru *lru, void *buf,  in bpf_common_lru_populate()  argument
565 	struct bpf_lru_list *l = &lru->common_lru.lru_list;  in bpf_common_lru_populate()
579 static void bpf_percpu_lru_populate(struct bpf_lru *lru, void *buf,  in bpf_percpu_lru_populate()  argument
594 		l = per_cpu_ptr(lru->percpu_lru, cpu);  in bpf_percpu_lru_populate()
610 void bpf_lru_populate(struct bpf_lru *lru, void *buf, u32 node_offset,  in bpf_lru_populate()  argument
613 	if (lru->percpu)  in bpf_lru_populate()
614 		bpf_percpu_lru_populate(lru, buf, node_offset, elem_size,  in bpf_lru_populate()
617 		bpf_common_lru_populate(lru, buf, node_offset, elem_size,  in bpf_lru_populate()
648 int bpf_lru_init(struct bpf_lru *lru, bool percpu, u32 hash_offset,  in bpf_lru_init()  argument
654 		lru->percpu_lru = alloc_percpu(struct bpf_lru_list);  in bpf_lru_init()
655 		if (!lru->percpu_lru)  in bpf_lru_init()
661 			l = per_cpu_ptr(lru->percpu_lru, cpu);  in bpf_lru_init()
664 		lru->nr_scans = PERCPU_NR_SCANS;  in bpf_lru_init()
666 		struct bpf_common_lru *clru = &lru->common_lru;  in bpf_lru_init()
680 		lru->nr_scans = LOCAL_NR_SCANS;  in bpf_lru_init()
683 	lru->percpu = percpu;  in bpf_lru_init()
684 	lru->del_from_htab = del_from_htab;  in bpf_lru_init()
685 	lru->del_arg = del_arg;  in bpf_lru_init()
686 	lru->hash_offset = hash_offset;  in bpf_lru_init()
691 void bpf_lru_destroy(struct bpf_lru *lru)  in bpf_lru_destroy()  argument
693 	if (lru->percpu)  in bpf_lru_destroy()
694 		free_percpu(lru->percpu_lru);  in bpf_lru_destroy()
696 		free_percpu(lru->common_lru.local_list);  in bpf_lru_destroy()