27 static int init_srcu_struct_fields(struct srcu_struct *ssp)  in init_srcu_struct_fields()  argument
29 	ssp->srcu_lock_nesting[0] = 0;  in init_srcu_struct_fields()
30 	ssp->srcu_lock_nesting[1] = 0;  in init_srcu_struct_fields()
31 	init_swait_queue_head(&ssp->srcu_wq);  in init_srcu_struct_fields()
32 	ssp->srcu_cb_head = NULL;  in init_srcu_struct_fields()
33 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in init_srcu_struct_fields()
34 	ssp->srcu_gp_running = false;  in init_srcu_struct_fields()
35 	ssp->srcu_gp_waiting = false;  in init_srcu_struct_fields()
36 	ssp->srcu_idx = 0;  in init_srcu_struct_fields()
37 	INIT_WORK(&ssp->srcu_work, srcu_drive_gp);  in init_srcu_struct_fields()
38 	INIT_LIST_HEAD(&ssp->srcu_work.entry);  in init_srcu_struct_fields()
44 int __init_srcu_struct(struct srcu_struct *ssp, const char *name,  in __init_srcu_struct()  argument
48 	debug_check_no_locks_freed((void *)ssp, sizeof(*ssp));  in __init_srcu_struct()
49 	lockdep_init_map(&ssp->dep_map, name, key, 0);  in __init_srcu_struct()
50 	return init_srcu_struct_fields(ssp);  in __init_srcu_struct()
58  * @ssp: structure to initialize.
64 int init_srcu_struct(struct srcu_struct *ssp)  in init_srcu_struct()  argument
66 	return init_srcu_struct_fields(ssp);  in init_srcu_struct()
74  * @ssp: structure to clean up.
79 void cleanup_srcu_struct(struct srcu_struct *ssp)  in cleanup_srcu_struct()  argument
81 	WARN_ON(ssp->srcu_lock_nesting[0] || ssp->srcu_lock_nesting[1]);  in cleanup_srcu_struct()
82 	flush_work(&ssp->srcu_work);  in cleanup_srcu_struct()
83 	WARN_ON(ssp->srcu_gp_running);  in cleanup_srcu_struct()
84 	WARN_ON(ssp->srcu_gp_waiting);  in cleanup_srcu_struct()
85 	WARN_ON(ssp->srcu_cb_head);  in cleanup_srcu_struct()
86 	WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);  in cleanup_srcu_struct()
94 void __srcu_read_unlock(struct srcu_struct *ssp, int idx)  in __srcu_read_unlock()  argument
96 	int newval = ssp->srcu_lock_nesting[idx] - 1;  in __srcu_read_unlock()
98 	WRITE_ONCE(ssp->srcu_lock_nesting[idx], newval);  in __srcu_read_unlock()
99 	if (!newval && READ_ONCE(ssp->srcu_gp_waiting))  in __srcu_read_unlock()
100 		swake_up_one(&ssp->srcu_wq);  in __srcu_read_unlock()
114 	struct srcu_struct *ssp;  in srcu_drive_gp()  local
116 	ssp = container_of(wp, struct srcu_struct, srcu_work);  in srcu_drive_gp()
117 	if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head))  in srcu_drive_gp()
121 	WRITE_ONCE(ssp->srcu_gp_running, true);  in srcu_drive_gp()
123 	lh = ssp->srcu_cb_head;  in srcu_drive_gp()
124 	ssp->srcu_cb_head = NULL;  in srcu_drive_gp()
125 	ssp->srcu_cb_tail = &ssp->srcu_cb_head;  in srcu_drive_gp()
127 	idx = ssp->srcu_idx;  in srcu_drive_gp()
128 	WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx);  in srcu_drive_gp()
129 	WRITE_ONCE(ssp->srcu_gp_waiting, true);  /* srcu_read_unlock() wakes! */  in srcu_drive_gp()
130 	swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));  in srcu_drive_gp()
131 	WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */  in srcu_drive_gp()
148 	WRITE_ONCE(ssp->srcu_gp_running, false);  in srcu_drive_gp()
149 	if (READ_ONCE(ssp->srcu_cb_head))  in srcu_drive_gp()
150 		schedule_work(&ssp->srcu_work);  in srcu_drive_gp()
158 void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,  in call_srcu()  argument
166 	*ssp->srcu_cb_tail = rhp;  in call_srcu()
167 	ssp->srcu_cb_tail = &rhp->next;  in call_srcu()
169 	if (!READ_ONCE(ssp->srcu_gp_running)) {  in call_srcu()
171 			schedule_work(&ssp->srcu_work);  in call_srcu()
172 		else if (list_empty(&ssp->srcu_work.entry))  in call_srcu()
173 			list_add(&ssp->srcu_work.entry, &srcu_boot_list);  in call_srcu()
181 void synchronize_srcu(struct srcu_struct *ssp)  in synchronize_srcu()  argument
187 	call_srcu(ssp, &rs.head, wakeme_after_rcu);  in synchronize_srcu()
206 	struct srcu_struct *ssp;  in srcu_init()  local
210 		ssp = list_first_entry(&srcu_boot_list,  in srcu_init()
212 		list_del_init(&ssp->srcu_work.entry);  in srcu_init()
213 		schedule_work(&ssp->srcu_work);  in srcu_init()