2812 	struct kfree_rcu_cpu *krcp;  member
2885 	struct kfree_rcu_cpu *krcp;  in krc_this_cpu_lock()  local
2888 	krcp = this_cpu_ptr(&krc);  in krc_this_cpu_lock()
2889 	raw_spin_lock(&krcp->lock);  in krc_this_cpu_lock()
2891 	return krcp;  in krc_this_cpu_lock()
2895 krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)  in krc_this_cpu_unlock()  argument
2897 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in krc_this_cpu_unlock()
2901 get_cached_bnode(struct kfree_rcu_cpu *krcp)  in get_cached_bnode()  argument
2903 	if (!krcp->nr_bkv_objs)  in get_cached_bnode()
2906 	WRITE_ONCE(krcp->nr_bkv_objs, krcp->nr_bkv_objs - 1);  in get_cached_bnode()
2908 		llist_del_first(&krcp->bkvcache);  in get_cached_bnode()
2912 put_cached_bnode(struct kfree_rcu_cpu *krcp,  in put_cached_bnode()  argument
2916 	if (krcp->nr_bkv_objs >= rcu_min_cached_objs)  in put_cached_bnode()
2919 	llist_add((struct llist_node *) bnode, &krcp->bkvcache);  in put_cached_bnode()
2920 	WRITE_ONCE(krcp->nr_bkv_objs, krcp->nr_bkv_objs + 1);  in put_cached_bnode()
2925 drain_page_cache(struct kfree_rcu_cpu *krcp)  in drain_page_cache()  argument
2934 	raw_spin_lock_irqsave(&krcp->lock, flags);  in drain_page_cache()
2935 	page_list = llist_del_all(&krcp->bkvcache);  in drain_page_cache()
2936 	WRITE_ONCE(krcp->nr_bkv_objs, 0);  in drain_page_cache()
2937 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in drain_page_cache()
2948 kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp,  in kvfree_rcu_bulk()  argument
2974 	raw_spin_lock_irqsave(&krcp->lock, flags);  in kvfree_rcu_bulk()
2975 	if (put_cached_bnode(krcp, bnode))  in kvfree_rcu_bulk()
2977 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in kvfree_rcu_bulk()
3017 	struct kfree_rcu_cpu *krcp;  in kfree_rcu_work()  local
3024 	krcp = krwp->krcp;  in kfree_rcu_work()
3026 	raw_spin_lock_irqsave(&krcp->lock, flags);  in kfree_rcu_work()
3035 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in kfree_rcu_work()
3041 			kvfree_rcu_bulk(krcp, bnode, i);  in kfree_rcu_work()
3056 need_offload_krc(struct kfree_rcu_cpu *krcp)  in need_offload_krc()  argument
3061 		if (!list_empty(&krcp->bulk_head[i]))  in need_offload_krc()
3064 	return !!READ_ONCE(krcp->head);  in need_offload_krc()
3079 static int krc_count(struct kfree_rcu_cpu *krcp)  in krc_count()  argument
3081 	int sum = atomic_read(&krcp->head_count);  in krc_count()
3085 		sum += atomic_read(&krcp->bulk_count[i]);  in krc_count()
3091 schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)  in schedule_delayed_monitor_work()  argument
3095 	delay = krc_count(krcp) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES;  in schedule_delayed_monitor_work()
3096 	if (delayed_work_pending(&krcp->monitor_work)) {  in schedule_delayed_monitor_work()
3097 		delay_left = krcp->monitor_work.timer.expires - jiffies;  in schedule_delayed_monitor_work()
3099 			mod_delayed_work(system_wq, &krcp->monitor_work, delay);  in schedule_delayed_monitor_work()
3102 	queue_delayed_work(system_wq, &krcp->monitor_work, delay);  in schedule_delayed_monitor_work()
3106 kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp)  in kvfree_rcu_drain_ready()  argument
3114 	raw_spin_lock_irqsave(&krcp->lock, flags);  in kvfree_rcu_drain_ready()
3118 		list_for_each_entry_safe_reverse(bnode, n, &krcp->bulk_head[i], list) {  in kvfree_rcu_drain_ready()
3122 			atomic_sub(bnode->nr_records, &krcp->bulk_count[i]);  in kvfree_rcu_drain_ready()
3127 	if (krcp->head && poll_state_synchronize_rcu(krcp->head_gp_snap)) {  in kvfree_rcu_drain_ready()
3128 		head_ready = krcp->head;  in kvfree_rcu_drain_ready()
3129 		atomic_set(&krcp->head_count, 0);  in kvfree_rcu_drain_ready()
3130 		WRITE_ONCE(krcp->head, NULL);  in kvfree_rcu_drain_ready()
3132 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in kvfree_rcu_drain_ready()
3136 			kvfree_rcu_bulk(krcp, bnode, i);  in kvfree_rcu_drain_ready()
3148 	struct kfree_rcu_cpu *krcp = container_of(work,  in kfree_rcu_monitor()  local
3154 	kvfree_rcu_drain_ready(krcp);  in kfree_rcu_monitor()
3156 	raw_spin_lock_irqsave(&krcp->lock, flags);  in kfree_rcu_monitor()
3160 		struct kfree_rcu_cpu_work *krwp = &(krcp->krw_arr[i]);  in kfree_rcu_monitor()
3169 		if (need_offload_krc(krcp)) {  in kfree_rcu_monitor()
3174 					atomic_set(&krcp->bulk_count[j], 0);  in kfree_rcu_monitor()
3175 					list_replace_init(&krcp->bulk_head[j],  in kfree_rcu_monitor()
3183 				krwp->head_free = krcp->head;  in kfree_rcu_monitor()
3185 				atomic_set(&krcp->head_count, 0);  in kfree_rcu_monitor()
3186 				WRITE_ONCE(krcp->head, NULL);  in kfree_rcu_monitor()
3198 	raw_spin_unlock_irqrestore(&krcp->lock, flags);  in kfree_rcu_monitor()
3205 	if (need_offload_krc(krcp))  in kfree_rcu_monitor()
3206 		schedule_delayed_monitor_work(krcp);  in kfree_rcu_monitor()
3212 	struct kfree_rcu_cpu *krcp =  in schedule_page_work_fn()  local
3215 	queue_delayed_work(system_highpri_wq, &krcp->page_cache_work, 0);  in schedule_page_work_fn()
3222 	struct kfree_rcu_cpu *krcp =  in fill_page_cache_func()  local
3230 	nr_pages = atomic_read(&krcp->backoff_page_cache_fill) ?  in fill_page_cache_func()
3233 	for (i = READ_ONCE(krcp->nr_bkv_objs); i < nr_pages; i++) {  in fill_page_cache_func()
3240 		raw_spin_lock_irqsave(&krcp->lock, flags);  in fill_page_cache_func()
3241 		pushed = put_cached_bnode(krcp, bnode);  in fill_page_cache_func()
3242 		raw_spin_unlock_irqrestore(&krcp->lock, flags);  in fill_page_cache_func()
3250 	atomic_set(&krcp->work_in_progress, 0);  in fill_page_cache_func()
3251 	atomic_set(&krcp->backoff_page_cache_fill, 0);  in fill_page_cache_func()
3255 run_page_cache_worker(struct kfree_rcu_cpu *krcp)  in run_page_cache_worker()  argument
3262 			!atomic_xchg(&krcp->work_in_progress, 1)) {  in run_page_cache_worker()
3263 		if (atomic_read(&krcp->backoff_page_cache_fill)) {  in run_page_cache_worker()
3265 				&krcp->page_cache_work,  in run_page_cache_worker()
3268 			hrtimer_init(&krcp->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in run_page_cache_worker()
3269 			krcp->hrtimer.function = schedule_page_work_fn;  in run_page_cache_worker()
3270 			hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);  in run_page_cache_worker()
3282 add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,  in add_ptr_to_bulk_krc_lock()  argument
3288 	*krcp = krc_this_cpu_lock(flags);  in add_ptr_to_bulk_krc_lock()
3289 	if (unlikely(!(*krcp)->initialized))  in add_ptr_to_bulk_krc_lock()
3293 	bnode = list_first_entry_or_null(&(*krcp)->bulk_head[idx],  in add_ptr_to_bulk_krc_lock()
3298 		bnode = get_cached_bnode(*krcp);  in add_ptr_to_bulk_krc_lock()
3300 			krc_this_cpu_unlock(*krcp, *flags);  in add_ptr_to_bulk_krc_lock()
3315 			raw_spin_lock_irqsave(&(*krcp)->lock, *flags);  in add_ptr_to_bulk_krc_lock()
3323 		list_add(&bnode->list, &(*krcp)->bulk_head[idx]);  in add_ptr_to_bulk_krc_lock()
3329 	atomic_inc(&(*krcp)->bulk_count[idx]);  in add_ptr_to_bulk_krc_lock()
3349 	struct kfree_rcu_cpu *krcp;  in kvfree_call_rcu()  local
3373 	success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);  in kvfree_call_rcu()
3375 		run_page_cache_worker(krcp);  in kvfree_call_rcu()
3382 		head->next = krcp->head;  in kvfree_call_rcu()
3383 		WRITE_ONCE(krcp->head, head);  in kvfree_call_rcu()
3384 		atomic_inc(&krcp->head_count);  in kvfree_call_rcu()
3387 		krcp->head_gp_snap = get_state_synchronize_rcu();  in kvfree_call_rcu()
3393 		schedule_delayed_monitor_work(krcp);  in kvfree_call_rcu()
3396 	krc_this_cpu_unlock(krcp, flags);  in kvfree_call_rcu()
3419 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);  in kfree_rcu_shrink_count()  local
3421 		count += krc_count(krcp);  in kfree_rcu_shrink_count()
3422 		count += READ_ONCE(krcp->nr_bkv_objs);  in kfree_rcu_shrink_count()
3423 		atomic_set(&krcp->backoff_page_cache_fill, 1);  in kfree_rcu_shrink_count()
3436 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);  in kfree_rcu_shrink_scan()  local
3438 		count = krc_count(krcp);  in kfree_rcu_shrink_scan()
3439 		count += drain_page_cache(krcp);  in kfree_rcu_shrink_scan()
3440 		kfree_rcu_monitor(&krcp->monitor_work.work);  in kfree_rcu_shrink_scan()
3464 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);  in kfree_rcu_scheduler_running()  local
3466 		if (need_offload_krc(krcp))  in kfree_rcu_scheduler_running()
3467 			schedule_delayed_monitor_work(krcp);  in kfree_rcu_scheduler_running()
4948 		struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);  in kfree_rcu_batch_init()  local
4951 			INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);  in kfree_rcu_batch_init()
4952 			krcp->krw_arr[i].krcp = krcp;  in kfree_rcu_batch_init()
4955 				INIT_LIST_HEAD(&krcp->krw_arr[i].bulk_head_free[j]);  in kfree_rcu_batch_init()
4959 			INIT_LIST_HEAD(&krcp->bulk_head[i]);  in kfree_rcu_batch_init()
4961 		INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);  in kfree_rcu_batch_init()
4962 		INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func);  in kfree_rcu_batch_init()
4963 		krcp->initialized = true;  in kfree_rcu_batch_init()