40 	call_single_data_t	__percpu *csd;  member
65 	cfd->csd = alloc_percpu(call_single_data_t);  in smpcfd_prepare_cpu()
66 	if (!cfd->csd) {  in smpcfd_prepare_cpu()
81 	free_percpu(cfd->csd);  in smpcfd_dead_cpu()
130 csd_do_func(smp_call_func_t func, void *info, struct __call_single_data *csd)  in csd_do_func()  argument
132 	trace_csd_function_entry(func, csd);  in csd_do_func()
134 	trace_csd_function_exit(func, csd);  in csd_do_func()
177 static void __csd_lock_record(struct __call_single_data *csd)  in __csd_lock_record()  argument
179 	if (!csd) {  in __csd_lock_record()
184 	__this_cpu_write(cur_csd_func, csd->func);  in __csd_lock_record()
185 	__this_cpu_write(cur_csd_info, csd->info);  in __csd_lock_record()
187 	__this_cpu_write(cur_csd, csd);  in __csd_lock_record()
192 static __always_inline void csd_lock_record(struct __call_single_data *csd)  in csd_lock_record()  argument
195 		__csd_lock_record(csd);  in csd_lock_record()
198 static int csd_lock_wait_getcpu(struct __call_single_data *csd)  in csd_lock_wait_getcpu()  argument
202 	csd_type = CSD_TYPE(csd);  in csd_lock_wait_getcpu()
204 		return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */  in csd_lock_wait_getcpu()
213 static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id)  in csd_lock_wait_toolong()  argument
220 	unsigned int flags = READ_ONCE(csd->node.u_flags);  in csd_lock_wait_toolong()
226 		cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
240 	cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
248 		 cpu, csd->func, csd->info);  in csd_lock_wait_toolong()
249 	if (cpu_cur_csd && csd != cpu_cur_csd) {  in csd_lock_wait_toolong()
279 static void __csd_lock_wait(struct __call_single_data *csd)  in __csd_lock_wait()  argument
286 		if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))  in __csd_lock_wait()
293 static __always_inline void csd_lock_wait(struct __call_single_data *csd)  in csd_lock_wait()  argument
296 		__csd_lock_wait(csd);  in csd_lock_wait()
300 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
303 static void csd_lock_record(struct __call_single_data *csd)  in csd_lock_record()  argument
307 static __always_inline void csd_lock_wait(struct __call_single_data *csd)  in csd_lock_wait()  argument
309 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
313 static __always_inline void csd_lock(struct __call_single_data *csd)  in csd_lock()  argument
315 	csd_lock_wait(csd);  in csd_lock()
316 	csd->node.u_flags |= CSD_FLAG_LOCK;  in csd_lock()
326 static __always_inline void csd_unlock(struct __call_single_data *csd)  in csd_unlock()  argument
328 	WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));  in csd_unlock()
333 	smp_store_release(&csd->node.u_flags, 0);  in csd_unlock()
348 		call_single_data_t *csd;  in __smp_call_single_queue()  local
351 		csd = container_of(node, call_single_data_t, node.llist);  in __smp_call_single_queue()
352 		func = CSD_TYPE(csd) == CSD_TYPE_TTWU ?  in __smp_call_single_queue()
353 			sched_ttwu_pending : csd->func;  in __smp_call_single_queue()
355 		trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);  in __smp_call_single_queue()
379 static int generic_exec_single(int cpu, struct __call_single_data *csd)  in generic_exec_single()  argument
382 		smp_call_func_t func = csd->func;  in generic_exec_single()
383 		void *info = csd->info;  in generic_exec_single()
390 		csd_lock_record(csd);  in generic_exec_single()
391 		csd_unlock(csd);  in generic_exec_single()
400 		csd_unlock(csd);  in generic_exec_single()
404 	__smp_call_single_queue(cpu, &csd->node.llist);  in generic_exec_single()
436 	call_single_data_t *csd, *csd_next;  in __flush_smp_call_function_queue()  local
462 		llist_for_each_entry(csd, entry, node.llist) {  in __flush_smp_call_function_queue()
463 			switch (CSD_TYPE(csd)) {  in __flush_smp_call_function_queue()
468 					csd->func);  in __flush_smp_call_function_queue()
477 					CSD_TYPE(csd));  in __flush_smp_call_function_queue()
487 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
489 		if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {  in __flush_smp_call_function_queue()
490 			smp_call_func_t func = csd->func;  in __flush_smp_call_function_queue()
491 			void *info = csd->info;  in __flush_smp_call_function_queue()
499 			csd_lock_record(csd);  in __flush_smp_call_function_queue()
500 			csd_do_func(func, info, csd);  in __flush_smp_call_function_queue()
501 			csd_unlock(csd);  in __flush_smp_call_function_queue()
504 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
515 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
516 		int type = CSD_TYPE(csd);  in __flush_smp_call_function_queue()
526 				smp_call_func_t func = csd->func;  in __flush_smp_call_function_queue()
527 				void *info = csd->info;  in __flush_smp_call_function_queue()
529 				csd_lock_record(csd);  in __flush_smp_call_function_queue()
530 				csd_unlock(csd);  in __flush_smp_call_function_queue()
531 				csd_do_func(func, info, csd);  in __flush_smp_call_function_queue()
534 				irq_work_single(csd);  in __flush_smp_call_function_queue()
538 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
546 		csd = llist_entry(entry, typeof(*csd), node.llist);  in __flush_smp_call_function_queue()
547 		csd_do_func(sched_ttwu_pending, entry, csd);  in __flush_smp_call_function_queue()
593 	call_single_data_t *csd;  in smp_call_function_single()  local
623 	csd = &csd_stack;  in smp_call_function_single()
625 		csd = this_cpu_ptr(&csd_data);  in smp_call_function_single()
626 		csd_lock(csd);  in smp_call_function_single()
629 	csd->func = func;  in smp_call_function_single()
630 	csd->info = info;  in smp_call_function_single()
632 	csd->node.src = smp_processor_id();  in smp_call_function_single()
633 	csd->node.dst = cpu;  in smp_call_function_single()
636 	err = generic_exec_single(cpu, csd);  in smp_call_function_single()
639 		csd_lock_wait(csd);  in smp_call_function_single()
670 int smp_call_function_single_async(int cpu, struct __call_single_data *csd)  in smp_call_function_single_async()  argument
676 	if (csd->node.u_flags & CSD_FLAG_LOCK) {  in smp_call_function_single_async()
681 	csd->node.u_flags = CSD_FLAG_LOCK;  in smp_call_function_single_async()
684 	err = generic_exec_single(cpu, csd);  in smp_call_function_single_async()
795 			call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);  in smp_call_function_many_cond()  local
802 			csd_lock(csd);  in smp_call_function_many_cond()
804 				csd->node.u_flags |= CSD_TYPE_SYNC;  in smp_call_function_many_cond()
805 			csd->func = func;  in smp_call_function_many_cond()
806 			csd->info = info;  in smp_call_function_many_cond()
808 			csd->node.src = smp_processor_id();  in smp_call_function_many_cond()
809 			csd->node.dst = cpu;  in smp_call_function_many_cond()
811 			trace_csd_queue_cpu(cpu, _RET_IP_, func, csd);  in smp_call_function_many_cond()
813 			if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {  in smp_call_function_many_cond()
841 			call_single_data_t *csd;  in smp_call_function_many_cond()  local
843 			csd = per_cpu_ptr(cfd->csd, cpu);  in smp_call_function_many_cond()
844 			csd_lock_wait(csd);  in smp_call_function_many_cond()