15 typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);
20 typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
132 static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate)  in set_tasks_gp_state()  argument
134 	rtp->gp_state = newstate;  in set_tasks_gp_state()
135 	rtp->gp_jiffies = jiffies;  in set_tasks_gp_state()
140 static const char *tasks_gp_state_getname(struct rcu_tasks *rtp)  in tasks_gp_state_getname()  argument
142 	int i = data_race(rtp->gp_state); // Let KCSAN detect update races  in tasks_gp_state_getname()
153 				   struct rcu_tasks *rtp)  in call_rcu_tasks_generic()  argument
160 	raw_spin_lock_irqsave(&rtp->cbs_lock, flags);  in call_rcu_tasks_generic()
161 	needwake = !rtp->cbs_head;  in call_rcu_tasks_generic()
162 	WRITE_ONCE(*rtp->cbs_tail, rhp);  in call_rcu_tasks_generic()
163 	rtp->cbs_tail = &rhp->next;  in call_rcu_tasks_generic()
164 	raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);  in call_rcu_tasks_generic()
166 	if (needwake && READ_ONCE(rtp->kthread_ptr))  in call_rcu_tasks_generic()
167 		wake_up(&rtp->cbs_wq);  in call_rcu_tasks_generic()
171 static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp)  in synchronize_rcu_tasks_generic()  argument
178 	wait_rcu_gp(rtp->call_func);  in synchronize_rcu_tasks_generic()
187 	struct rcu_tasks *rtp = arg;  in rcu_tasks_kthread()  local
191 	WRITE_ONCE(rtp->kthread_ptr, current); // Let GPs start!  in rcu_tasks_kthread()
202 		raw_spin_lock_irqsave(&rtp->cbs_lock, flags);  in rcu_tasks_kthread()
204 		list = rtp->cbs_head;  in rcu_tasks_kthread()
205 		rtp->cbs_head = NULL;  in rcu_tasks_kthread()
206 		rtp->cbs_tail = &rtp->cbs_head;  in rcu_tasks_kthread()
207 		raw_spin_unlock_irqrestore(&rtp->cbs_lock, flags);  in rcu_tasks_kthread()
211 			wait_event_interruptible(rtp->cbs_wq,  in rcu_tasks_kthread()
212 						 READ_ONCE(rtp->cbs_head));  in rcu_tasks_kthread()
213 			if (!rtp->cbs_head) {  in rcu_tasks_kthread()
215 				set_tasks_gp_state(rtp, RTGS_WAIT_WAIT_CBS);  in rcu_tasks_kthread()
222 		set_tasks_gp_state(rtp, RTGS_WAIT_GP);  in rcu_tasks_kthread()
223 		rtp->gp_start = jiffies;  in rcu_tasks_kthread()
224 		rtp->gp_func(rtp);  in rcu_tasks_kthread()
225 		rtp->n_gps++;  in rcu_tasks_kthread()
228 		set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);  in rcu_tasks_kthread()
238 		schedule_timeout_idle(rtp->gp_sleep);  in rcu_tasks_kthread()
240 		set_tasks_gp_state(rtp, RTGS_WAIT_CBS);  in rcu_tasks_kthread()
245 static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp)  in rcu_spawn_tasks_kthread_generic()  argument
249 	t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname);  in rcu_spawn_tasks_kthread_generic()
250 …%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name))  in rcu_spawn_tasks_kthread_generic()
281 static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)  in show_rcu_tasks_generic_gp_kthread()  argument
284 		rtp->kname,  in show_rcu_tasks_generic_gp_kthread()
285 		tasks_gp_state_getname(rtp), data_race(rtp->gp_state),  in show_rcu_tasks_generic_gp_kthread()
286 		jiffies - data_race(rtp->gp_jiffies),  in show_rcu_tasks_generic_gp_kthread()
287 		data_race(rtp->n_gps),  in show_rcu_tasks_generic_gp_kthread()
288 		data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis),  in show_rcu_tasks_generic_gp_kthread()
289 		".k"[!!data_race(rtp->kthread_ptr)],  in show_rcu_tasks_generic_gp_kthread()
290 		".C"[!!data_race(rtp->cbs_head)],  in show_rcu_tasks_generic_gp_kthread()
304 static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)  in rcu_tasks_wait_gp()  argument
311 	set_tasks_gp_state(rtp, RTGS_PRE_WAIT_GP);  in rcu_tasks_wait_gp()
312 	rtp->pregp_func();  in rcu_tasks_wait_gp()
320 	set_tasks_gp_state(rtp, RTGS_SCAN_TASKLIST);  in rcu_tasks_wait_gp()
323 		rtp->pertask_func(t, &holdouts);  in rcu_tasks_wait_gp()
326 	set_tasks_gp_state(rtp, RTGS_POST_SCAN_TASKLIST);  in rcu_tasks_wait_gp()
327 	rtp->postscan_func(&holdouts);  in rcu_tasks_wait_gp()
337 	fract = rtp->init_fract;  in rcu_tasks_wait_gp()
350 		set_tasks_gp_state(rtp, RTGS_WAIT_SCAN_HOLDOUTS);  in rcu_tasks_wait_gp()
362 		set_tasks_gp_state(rtp, RTGS_SCAN_HOLDOUTS);  in rcu_tasks_wait_gp()
363 		rtp->holdouts_func(&holdouts, needreport, &firstreport);  in rcu_tasks_wait_gp()
366 	set_tasks_gp_state(rtp, RTGS_POST_GP);  in rcu_tasks_wait_gp()
367 	rtp->postgp_func(rtp);  in rcu_tasks_wait_gp()
474 static void rcu_tasks_postgp(struct rcu_tasks *rtp)  in rcu_tasks_postgp()  argument
623 static void rcu_tasks_rude_wait_gp(struct rcu_tasks *rtp)  in rcu_tasks_rude_wait_gp()  argument
625 	rtp->n_ipis += cpumask_weight(cpu_online_mask);  in rcu_tasks_rude_wait_gp()
1069 static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp)  in rcu_tasks_trace_postgp()  argument
1082 	set_tasks_gp_state(rtp, RTGS_WAIT_READERS);  in rcu_tasks_trace_postgp()