66 void rcu_sync_lockdep_assert(struct rcu_sync *rsp)  in rcu_sync_lockdep_assert()  argument
68 	RCU_LOCKDEP_WARN(!gp_ops[rsp->gp_type].held(),  in rcu_sync_lockdep_assert()
80 void rcu_sync_init(struct rcu_sync *rsp, enum rcu_sync_type type)  in rcu_sync_init()  argument
82 	memset(rsp, 0, sizeof(*rsp));  in rcu_sync_init()
83 	init_waitqueue_head(&rsp->gp_wait);  in rcu_sync_init()
84 	rsp->gp_type = type;  in rcu_sync_init()
96 void rcu_sync_enter_start(struct rcu_sync *rsp)  in rcu_sync_enter_start()  argument
98 	rsp->gp_count++;  in rcu_sync_enter_start()
99 	rsp->gp_state = GP_PASSED;  in rcu_sync_enter_start()
117 void rcu_sync_enter(struct rcu_sync *rsp)  in rcu_sync_enter()  argument
121 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_enter()
122 	need_wait = rsp->gp_count++;  in rcu_sync_enter()
123 	need_sync = rsp->gp_state == GP_IDLE;  in rcu_sync_enter()
125 		rsp->gp_state = GP_PENDING;  in rcu_sync_enter()
126 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_enter()
131 		gp_ops[rsp->gp_type].sync();  in rcu_sync_enter()
132 		rsp->gp_state = GP_PASSED;  in rcu_sync_enter()
133 		wake_up_all(&rsp->gp_wait);  in rcu_sync_enter()
135 		wait_event(rsp->gp_wait, rsp->gp_state == GP_PASSED);  in rcu_sync_enter()
142 		BUG_ON(rsp->gp_state != GP_PASSED);  in rcu_sync_enter()
166 	struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head);  in rcu_sync_func()  local
169 	BUG_ON(rsp->gp_state != GP_PASSED);  in rcu_sync_func()
170 	BUG_ON(rsp->cb_state == CB_IDLE);  in rcu_sync_func()
172 	spin_lock_irqsave(&rsp->rss_lock, flags);  in rcu_sync_func()
173 	if (rsp->gp_count) {  in rcu_sync_func()
177 		rsp->cb_state = CB_IDLE;  in rcu_sync_func()
178 	} else if (rsp->cb_state == CB_REPLAY) {  in rcu_sync_func()
183 		rsp->cb_state = CB_PENDING;  in rcu_sync_func()
184 		gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);  in rcu_sync_func()
190 		rsp->cb_state = CB_IDLE;  in rcu_sync_func()
191 		rsp->gp_state = GP_IDLE;  in rcu_sync_func()
193 	spin_unlock_irqrestore(&rsp->rss_lock, flags);  in rcu_sync_func()
206 void rcu_sync_exit(struct rcu_sync *rsp)  in rcu_sync_exit()  argument
208 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_exit()
209 	if (!--rsp->gp_count) {  in rcu_sync_exit()
210 		if (rsp->cb_state == CB_IDLE) {  in rcu_sync_exit()
211 			rsp->cb_state = CB_PENDING;  in rcu_sync_exit()
212 			gp_ops[rsp->gp_type].call(&rsp->cb_head, rcu_sync_func);  in rcu_sync_exit()
213 		} else if (rsp->cb_state == CB_PENDING) {  in rcu_sync_exit()
214 			rsp->cb_state = CB_REPLAY;  in rcu_sync_exit()
217 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_exit()
224 void rcu_sync_dtor(struct rcu_sync *rsp)  in rcu_sync_dtor()  argument
228 	BUG_ON(rsp->gp_count);  in rcu_sync_dtor()
230 	spin_lock_irq(&rsp->rss_lock);  in rcu_sync_dtor()
231 	if (rsp->cb_state == CB_REPLAY)  in rcu_sync_dtor()
232 		rsp->cb_state = CB_PENDING;  in rcu_sync_dtor()
233 	cb_state = rsp->cb_state;  in rcu_sync_dtor()
234 	spin_unlock_irq(&rsp->rss_lock);  in rcu_sync_dtor()
237 		gp_ops[rsp->gp_type].wait();  in rcu_sync_dtor()
238 		BUG_ON(rsp->cb_state != CB_IDLE);  in rcu_sync_dtor()