45 static bool bpf_lru_node_is_ref(const struct bpf_lru_node *node)  in bpf_lru_node_is_ref()  argument
47 	return node->ref;  in bpf_lru_node_is_ref()
65 					struct bpf_lru_node *node,  in __bpf_lru_node_move_to_free()  argument
69 	if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)))  in __bpf_lru_node_move_to_free()
75 	if (&node->list == l->next_inactive_rotation)  in __bpf_lru_node_move_to_free()
78 	bpf_lru_list_count_dec(l, node->type);  in __bpf_lru_node_move_to_free()
80 	node->type = tgt_free_type;  in __bpf_lru_node_move_to_free()
81 	list_move(&node->list, free_list);  in __bpf_lru_node_move_to_free()
86 				   struct bpf_lru_node *node,  in __bpf_lru_node_move_in()  argument
89 	if (WARN_ON_ONCE(!IS_LOCAL_LIST_TYPE(node->type)) ||  in __bpf_lru_node_move_in()
94 	node->type = tgt_type;  in __bpf_lru_node_move_in()
95 	node->ref = 0;  in __bpf_lru_node_move_in()
96 	list_move(&node->list, &l->lists[tgt_type]);  in __bpf_lru_node_move_in()
104 				struct bpf_lru_node *node,  in __bpf_lru_node_move()  argument
107 	if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)) ||  in __bpf_lru_node_move()
111 	if (node->type != tgt_type) {  in __bpf_lru_node_move()
112 		bpf_lru_list_count_dec(l, node->type);  in __bpf_lru_node_move()
114 		node->type = tgt_type;  in __bpf_lru_node_move()
116 	node->ref = 0;  in __bpf_lru_node_move()
121 	if (&node->list == l->next_inactive_rotation)  in __bpf_lru_node_move()
124 	list_move(&node->list, &l->lists[tgt_type]);  in __bpf_lru_node_move()
146 	struct bpf_lru_node *node, *tmp_node, *first_node;  in __bpf_lru_list_rotate_active()  local
150 	list_for_each_entry_safe_reverse(node, tmp_node, active, list) {  in __bpf_lru_list_rotate_active()
151 		if (bpf_lru_node_is_ref(node))  in __bpf_lru_list_rotate_active()
152 			__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE);  in __bpf_lru_list_rotate_active()
154 			__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE);  in __bpf_lru_list_rotate_active()
156 		if (++i == lru->nr_scans || node == first_node)  in __bpf_lru_list_rotate_active()
174 	struct bpf_lru_node *node;  in __bpf_lru_list_rotate_inactive()  local
191 		node = list_entry(cur, struct bpf_lru_node, list);  in __bpf_lru_list_rotate_inactive()
193 		if (bpf_lru_node_is_ref(node))  in __bpf_lru_list_rotate_inactive()
194 			__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE);  in __bpf_lru_list_rotate_inactive()
216 	struct bpf_lru_node *node, *tmp_node;  in __bpf_lru_list_shrink_inactive()  local
220 	list_for_each_entry_safe_reverse(node, tmp_node, inactive, list) {  in __bpf_lru_list_shrink_inactive()
221 		if (bpf_lru_node_is_ref(node)) {  in __bpf_lru_list_shrink_inactive()
222 			__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_ACTIVE);  in __bpf_lru_list_shrink_inactive()
223 		} else if (lru->del_from_htab(lru->del_arg, node)) {  in __bpf_lru_list_shrink_inactive()
224 			__bpf_lru_node_move_to_free(l, node, free_list,  in __bpf_lru_list_shrink_inactive()
265 	struct bpf_lru_node *node, *tmp_node;  in __bpf_lru_list_shrink()  local
280 	list_for_each_entry_safe_reverse(node, tmp_node, force_shrink_list,  in __bpf_lru_list_shrink()
282 		if (lru->del_from_htab(lru->del_arg, node)) {  in __bpf_lru_list_shrink()
283 			__bpf_lru_node_move_to_free(l, node, free_list,  in __bpf_lru_list_shrink()
296 	struct bpf_lru_node *node, *tmp_node;  in __local_list_flush()  local
298 	list_for_each_entry_safe_reverse(node, tmp_node,  in __local_list_flush()
300 		if (bpf_lru_node_is_ref(node))  in __local_list_flush()
301 			__bpf_lru_node_move_in(l, node, BPF_LRU_LIST_T_ACTIVE);  in __local_list_flush()
303 			__bpf_lru_node_move_in(l, node,  in __local_list_flush()
309 				   struct bpf_lru_node *node)  in bpf_lru_list_push_free()  argument
313 	if (WARN_ON_ONCE(IS_LOCAL_LIST_TYPE(node->type)))  in bpf_lru_list_push_free()
317 	__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE);  in bpf_lru_list_push_free()
325 	struct bpf_lru_node *node, *tmp_node;  in bpf_lru_list_pop_free_to_local()  local
334 	list_for_each_entry_safe(node, tmp_node, &l->lists[BPF_LRU_LIST_T_FREE],  in bpf_lru_list_pop_free_to_local()
336 		__bpf_lru_node_move_to_free(l, node, local_free_list(loc_l),  in bpf_lru_list_pop_free_to_local()
353 				     struct bpf_lru_node *node,  in __local_list_add_pending()  argument
356 	*(u32 *)((void *)node + lru->hash_offset) = hash;  in __local_list_add_pending()
357 	node->cpu = cpu;  in __local_list_add_pending()
358 	node->type = BPF_LRU_LOCAL_LIST_T_PENDING;  in __local_list_add_pending()
359 	node->ref = 0;  in __local_list_add_pending()
360 	list_add(&node->list, local_pending_list(loc_l));  in __local_list_add_pending()
366 	struct bpf_lru_node *node;  in __local_list_pop_free()  local
368 	node = list_first_entry_or_null(local_free_list(loc_l),  in __local_list_pop_free()
371 	if (node)  in __local_list_pop_free()
372 		list_del(&node->list);  in __local_list_pop_free()
374 	return node;  in __local_list_pop_free()
380 	struct bpf_lru_node *node;  in __local_list_pop_pending()  local
385 	list_for_each_entry_reverse(node, local_pending_list(loc_l),  in __local_list_pop_pending()
387 		if ((!bpf_lru_node_is_ref(node) || force) &&  in __local_list_pop_pending()
388 		    lru->del_from_htab(lru->del_arg, node)) {  in __local_list_pop_pending()
389 			list_del(&node->list);  in __local_list_pop_pending()
390 			return node;  in __local_list_pop_pending()
406 	struct bpf_lru_node *node = NULL;  in bpf_percpu_lru_pop_free()  local
423 		node = list_first_entry(free_list, struct bpf_lru_node, list);  in bpf_percpu_lru_pop_free()
424 		*(u32 *)((void *)node + lru->hash_offset) = hash;  in bpf_percpu_lru_pop_free()
425 		node->ref = 0;  in bpf_percpu_lru_pop_free()
426 		__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_INACTIVE);  in bpf_percpu_lru_pop_free()
431 	return node;  in bpf_percpu_lru_pop_free()
439 	struct bpf_lru_node *node;  in bpf_common_lru_pop_free()  local
448 	node = __local_list_pop_free(loc_l);  in bpf_common_lru_pop_free()
449 	if (!node) {  in bpf_common_lru_pop_free()
451 		node = __local_list_pop_free(loc_l);  in bpf_common_lru_pop_free()
454 	if (node)  in bpf_common_lru_pop_free()
455 		__local_list_add_pending(lru, loc_l, cpu, node, hash);  in bpf_common_lru_pop_free()
459 	if (node)  in bpf_common_lru_pop_free()
460 		return node;  in bpf_common_lru_pop_free()
477 		node = __local_list_pop_free(steal_loc_l);  in bpf_common_lru_pop_free()
478 		if (!node)  in bpf_common_lru_pop_free()
479 			node = __local_list_pop_pending(lru, steal_loc_l);  in bpf_common_lru_pop_free()
484 	} while (!node && steal != first_steal);  in bpf_common_lru_pop_free()
488 	if (node) {  in bpf_common_lru_pop_free()
490 		__local_list_add_pending(lru, loc_l, cpu, node, hash);  in bpf_common_lru_pop_free()
494 	return node;  in bpf_common_lru_pop_free()
506 				     struct bpf_lru_node *node)  in bpf_common_lru_push_free()  argument
510 	if (WARN_ON_ONCE(node->type == BPF_LRU_LIST_T_FREE) ||  in bpf_common_lru_push_free()
511 	    WARN_ON_ONCE(node->type == BPF_LRU_LOCAL_LIST_T_FREE))  in bpf_common_lru_push_free()
514 	if (node->type == BPF_LRU_LOCAL_LIST_T_PENDING) {  in bpf_common_lru_push_free()
517 		loc_l = per_cpu_ptr(lru->common_lru.local_list, node->cpu);  in bpf_common_lru_push_free()
521 		if (unlikely(node->type != BPF_LRU_LOCAL_LIST_T_PENDING)) {  in bpf_common_lru_push_free()
526 		node->type = BPF_LRU_LOCAL_LIST_T_FREE;  in bpf_common_lru_push_free()
527 		node->ref = 0;  in bpf_common_lru_push_free()
528 		list_move(&node->list, local_free_list(loc_l));  in bpf_common_lru_push_free()
535 	bpf_lru_list_push_free(&lru->common_lru.lru_list, node);  in bpf_common_lru_push_free()
539 				     struct bpf_lru_node *node)  in bpf_percpu_lru_push_free()  argument
544 	l = per_cpu_ptr(lru->percpu_lru, node->cpu);  in bpf_percpu_lru_push_free()
548 	__bpf_lru_node_move(l, node, BPF_LRU_LIST_T_FREE);  in bpf_percpu_lru_push_free()
553 void bpf_lru_push_free(struct bpf_lru *lru, struct bpf_lru_node *node)  in bpf_lru_push_free()  argument
556 		bpf_percpu_lru_push_free(lru, node);  in bpf_lru_push_free()
558 		bpf_common_lru_push_free(lru, node);  in bpf_lru_push_free()
569 		struct bpf_lru_node *node;  in bpf_common_lru_populate()  local
571 		node = (struct bpf_lru_node *)(buf + node_offset);  in bpf_common_lru_populate()
572 		node->type = BPF_LRU_LIST_T_FREE;  in bpf_common_lru_populate()
573 		node->ref = 0;  in bpf_common_lru_populate()
574 		list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]);  in bpf_common_lru_populate()
592 		struct bpf_lru_node *node;  in bpf_percpu_lru_populate()  local
596 		node = (struct bpf_lru_node *)(buf + node_offset);  in bpf_percpu_lru_populate()
597 		node->cpu = cpu;  in bpf_percpu_lru_populate()
598 		node->type = BPF_LRU_LIST_T_FREE;  in bpf_percpu_lru_populate()
599 		node->ref = 0;  in bpf_percpu_lru_populate()
600 		list_add(&node->list, &l->lists[BPF_LRU_LIST_T_FREE]);  in bpf_percpu_lru_populate()