81 	call_single_data_t	csd;  member
216 static void __csd_lock_record(struct __call_single_data *csd)  in __csd_lock_record()  argument
218 	if (!csd) {  in __csd_lock_record()
223 	__this_cpu_write(cur_csd_func, csd->func);  in __csd_lock_record()
224 	__this_cpu_write(cur_csd_info, csd->info);  in __csd_lock_record()
226 	__this_cpu_write(cur_csd, csd);  in __csd_lock_record()
231 static __always_inline void csd_lock_record(struct __call_single_data *csd)  in csd_lock_record()  argument
234 		__csd_lock_record(csd);  in csd_lock_record()
237 static int csd_lock_wait_getcpu(struct __call_single_data *csd)  in csd_lock_wait_getcpu()  argument
241 	csd_type = CSD_TYPE(csd);  in csd_lock_wait_getcpu()
243 		return csd->node.dst; /* Other CSD_TYPE_ values might not have ->dst. */  in csd_lock_wait_getcpu()
287 static void csd_lock_print_extended(struct __call_single_data *csd, int cpu)  in csd_lock_print_extended()  argument
290 	unsigned int srccpu = csd->node.src;  in csd_lock_print_extended()
326 static bool csd_lock_wait_toolong(struct __call_single_data *csd, u64 ts0, u64 *ts1, int *bug_id)  in csd_lock_wait_toolong()  argument
333 	unsigned int flags = READ_ONCE(csd->node.u_flags);  in csd_lock_wait_toolong()
339 		cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
353 	cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
361 		 cpu, csd->func, csd->info);  in csd_lock_wait_toolong()
362 	if (cpu_cur_csd && csd != cpu_cur_csd) {  in csd_lock_wait_toolong()
372 			csd_lock_print_extended(csd, cpu);  in csd_lock_wait_toolong()
392 static void __csd_lock_wait(struct __call_single_data *csd)  in __csd_lock_wait()  argument
399 		if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))  in __csd_lock_wait()
406 static __always_inline void csd_lock_wait(struct __call_single_data *csd)  in csd_lock_wait()  argument
409 		__csd_lock_wait(csd);  in csd_lock_wait()
413 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
436 static void csd_lock_record(struct __call_single_data *csd)  in csd_lock_record()  argument
440 static __always_inline void csd_lock_wait(struct __call_single_data *csd)  in csd_lock_wait()  argument
442 	smp_cond_load_acquire(&csd->node.u_flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
446 static __always_inline void csd_lock(struct __call_single_data *csd)  in csd_lock()  argument
448 	csd_lock_wait(csd);  in csd_lock()
449 	csd->node.u_flags |= CSD_FLAG_LOCK;  in csd_lock()
459 static __always_inline void csd_unlock(struct __call_single_data *csd)  in csd_unlock()  argument
461 	WARN_ON(!(csd->node.u_flags & CSD_FLAG_LOCK));  in csd_unlock()
466 	smp_store_release(&csd->node.u_flags, 0);  in csd_unlock()
506 static int generic_exec_single(int cpu, struct __call_single_data *csd)  in generic_exec_single()  argument
509 		smp_call_func_t func = csd->func;  in generic_exec_single()
510 		void *info = csd->info;  in generic_exec_single()
517 		csd_lock_record(csd);  in generic_exec_single()
518 		csd_unlock(csd);  in generic_exec_single()
527 		csd_unlock(csd);  in generic_exec_single()
531 	__smp_call_single_queue(cpu, &csd->node.llist);  in generic_exec_single()
565 	call_single_data_t *csd, *csd_next;  in __flush_smp_call_function_queue()  local
592 		llist_for_each_entry(csd, entry, node.llist) {  in __flush_smp_call_function_queue()
593 			switch (CSD_TYPE(csd)) {  in __flush_smp_call_function_queue()
598 					csd->func);  in __flush_smp_call_function_queue()
607 					CSD_TYPE(csd));  in __flush_smp_call_function_queue()
617 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
619 		if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {  in __flush_smp_call_function_queue()
620 			smp_call_func_t func = csd->func;  in __flush_smp_call_function_queue()
621 			void *info = csd->info;  in __flush_smp_call_function_queue()
629 			csd_lock_record(csd);  in __flush_smp_call_function_queue()
631 			csd_unlock(csd);  in __flush_smp_call_function_queue()
634 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
649 	llist_for_each_entry_safe(csd, csd_next, entry, node.llist) {  in __flush_smp_call_function_queue()
650 		int type = CSD_TYPE(csd);  in __flush_smp_call_function_queue()
660 				smp_call_func_t func = csd->func;  in __flush_smp_call_function_queue()
661 				void *info = csd->info;  in __flush_smp_call_function_queue()
663 				csd_lock_record(csd);  in __flush_smp_call_function_queue()
664 				csd_unlock(csd);  in __flush_smp_call_function_queue()
668 				irq_work_single(csd);  in __flush_smp_call_function_queue()
672 			prev = &csd->node.llist;  in __flush_smp_call_function_queue()
730 	call_single_data_t *csd;  in smp_call_function_single()  local
760 	csd = &csd_stack;  in smp_call_function_single()
762 		csd = this_cpu_ptr(&csd_data);  in smp_call_function_single()
763 		csd_lock(csd);  in smp_call_function_single()
766 	csd->func = func;  in smp_call_function_single()
767 	csd->info = info;  in smp_call_function_single()
769 	csd->node.src = smp_processor_id();  in smp_call_function_single()
770 	csd->node.dst = cpu;  in smp_call_function_single()
773 	err = generic_exec_single(cpu, csd);  in smp_call_function_single()
776 		csd_lock_wait(csd);  in smp_call_function_single()
807 int smp_call_function_single_async(int cpu, struct __call_single_data *csd)  in smp_call_function_single_async()  argument
813 	if (csd->node.u_flags & CSD_FLAG_LOCK) {  in smp_call_function_single_async()
818 	csd->node.u_flags = CSD_FLAG_LOCK;  in smp_call_function_single_async()
821 	err = generic_exec_single(cpu, csd);  in smp_call_function_single_async()
933 			call_single_data_t *csd = &pcpu->csd;  in smp_call_function_many_cond()  local
938 			csd_lock(csd);  in smp_call_function_many_cond()
940 				csd->node.u_flags |= CSD_TYPE_SYNC;  in smp_call_function_many_cond()
941 			csd->func = func;  in smp_call_function_many_cond()
942 			csd->info = info;  in smp_call_function_many_cond()
944 			csd->node.src = smp_processor_id();  in smp_call_function_many_cond()
945 			csd->node.dst = cpu;  in smp_call_function_many_cond()
948 			if (llist_add(&csd->node.llist, &per_cpu(call_single_queue, cpu))) {  in smp_call_function_many_cond()
984 			call_single_data_t *csd;  in smp_call_function_many_cond()  local
986 			csd = &per_cpu_ptr(cfd->pcpu, cpu)->csd;  in smp_call_function_many_cond()
987 			csd_lock_wait(csd);  in smp_call_function_many_cond()