33 	call_single_data_t	__percpu *csd;  member
56 	cfd->csd = alloc_percpu(call_single_data_t);  in smpcfd_prepare_cpu()
57 	if (!cfd->csd) {  in smpcfd_prepare_cpu()
72 	free_percpu(cfd->csd);  in smpcfd_dead_cpu()
112 static void csd_lock_record(call_single_data_t *csd)  in csd_lock_record()  argument
114 	if (!csd) {  in csd_lock_record()
119 	__this_cpu_write(cur_csd_func, csd->func);  in csd_lock_record()
120 	__this_cpu_write(cur_csd_info, csd->info);  in csd_lock_record()
122 	__this_cpu_write(cur_csd, csd);  in csd_lock_record()
127 static __always_inline int csd_lock_wait_getcpu(call_single_data_t *csd)  in csd_lock_wait_getcpu()  argument
131 	csd_type = CSD_TYPE(csd);  in csd_lock_wait_getcpu()
133 		return csd->dst; /* Other CSD_TYPE_ values might not have ->dst. */  in csd_lock_wait_getcpu()
142 static __always_inline bool csd_lock_wait_toolong(call_single_data_t *csd, u64 ts0, u64 *ts1, int *…  in csd_lock_wait_toolong()  argument
149 	unsigned int flags = READ_ONCE(csd->flags);  in csd_lock_wait_toolong()
154 		cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
168 	cpu = csd_lock_wait_getcpu(csd);  in csd_lock_wait_toolong()
176 		 cpu, csd->func, csd->info);  in csd_lock_wait_toolong()
177 	if (cpu_cur_csd && csd != cpu_cur_csd) {  in csd_lock_wait_toolong()
206 static __always_inline void csd_lock_wait(call_single_data_t *csd)  in csd_lock_wait()  argument
213 		if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id))  in csd_lock_wait()
221 static void csd_lock_record(call_single_data_t *csd)  in csd_lock_record()  argument
225 static __always_inline void csd_lock_wait(call_single_data_t *csd)  in csd_lock_wait()  argument
227 	smp_cond_load_acquire(&csd->flags, !(VAL & CSD_FLAG_LOCK));  in csd_lock_wait()
231 static __always_inline void csd_lock(call_single_data_t *csd)  in csd_lock()  argument
233 	csd_lock_wait(csd);  in csd_lock()
234 	csd->flags |= CSD_FLAG_LOCK;  in csd_lock()
244 static __always_inline void csd_unlock(call_single_data_t *csd)  in csd_unlock()  argument
246 	WARN_ON(!(csd->flags & CSD_FLAG_LOCK));  in csd_unlock()
251 	smp_store_release(&csd->flags, 0);  in csd_unlock()
278 static int generic_exec_single(int cpu, call_single_data_t *csd)  in generic_exec_single()  argument
281 		smp_call_func_t func = csd->func;  in generic_exec_single()
282 		void *info = csd->info;  in generic_exec_single()
289 		csd_lock_record(csd);  in generic_exec_single()
290 		csd_unlock(csd);  in generic_exec_single()
299 		csd_unlock(csd);  in generic_exec_single()
303 	__smp_call_single_queue(cpu, &csd->llist);  in generic_exec_single()
335 	call_single_data_t *csd, *csd_next;  in flush_smp_call_function_queue()  local
356 		llist_for_each_entry(csd, entry, llist) {  in flush_smp_call_function_queue()
357 			switch (CSD_TYPE(csd)) {  in flush_smp_call_function_queue()
362 					csd->func);  in flush_smp_call_function_queue()
371 					CSD_TYPE(csd));  in flush_smp_call_function_queue()
381 	llist_for_each_entry_safe(csd, csd_next, entry, llist) {  in flush_smp_call_function_queue()
383 		if (CSD_TYPE(csd) == CSD_TYPE_SYNC) {  in flush_smp_call_function_queue()
384 			smp_call_func_t func = csd->func;  in flush_smp_call_function_queue()
385 			void *info = csd->info;  in flush_smp_call_function_queue()
393 			csd_lock_record(csd);  in flush_smp_call_function_queue()
395 			csd_unlock(csd);  in flush_smp_call_function_queue()
398 			prev = &csd->llist;  in flush_smp_call_function_queue()
409 	llist_for_each_entry_safe(csd, csd_next, entry, llist) {  in flush_smp_call_function_queue()
410 		int type = CSD_TYPE(csd);  in flush_smp_call_function_queue()
420 				smp_call_func_t func = csd->func;  in flush_smp_call_function_queue()
421 				void *info = csd->info;  in flush_smp_call_function_queue()
423 				csd_lock_record(csd);  in flush_smp_call_function_queue()
424 				csd_unlock(csd);  in flush_smp_call_function_queue()
428 				irq_work_single(csd);  in flush_smp_call_function_queue()
432 			prev = &csd->llist;  in flush_smp_call_function_queue()
466 	call_single_data_t *csd;  in smp_call_function_single()  local
496 	csd = &csd_stack;  in smp_call_function_single()
498 		csd = this_cpu_ptr(&csd_data);  in smp_call_function_single()
499 		csd_lock(csd);  in smp_call_function_single()
502 	csd->func = func;  in smp_call_function_single()
503 	csd->info = info;  in smp_call_function_single()
505 	csd->src = smp_processor_id();  in smp_call_function_single()
506 	csd->dst = cpu;  in smp_call_function_single()
509 	err = generic_exec_single(cpu, csd);  in smp_call_function_single()
512 		csd_lock_wait(csd);  in smp_call_function_single()
541 int smp_call_function_single_async(int cpu, call_single_data_t *csd)  in smp_call_function_single_async()  argument
547 	if (csd->flags & CSD_FLAG_LOCK) {  in smp_call_function_single_async()
552 	csd->flags = CSD_FLAG_LOCK;  in smp_call_function_single_async()
555 	err = generic_exec_single(cpu, csd);  in smp_call_function_single_async()
663 		call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);  in smp_call_function_many_cond()  local
668 		csd_lock(csd);  in smp_call_function_many_cond()
670 			csd->flags |= CSD_TYPE_SYNC;  in smp_call_function_many_cond()
671 		csd->func = func;  in smp_call_function_many_cond()
672 		csd->info = info;  in smp_call_function_many_cond()
674 		csd->src = smp_processor_id();  in smp_call_function_many_cond()
675 		csd->dst = cpu;  in smp_call_function_many_cond()
677 		if (llist_add(&csd->llist, &per_cpu(call_single_queue, cpu)))  in smp_call_function_many_cond()
686 			call_single_data_t *csd;  in smp_call_function_many_cond()  local
688 			csd = per_cpu_ptr(cfd->csd, cpu);  in smp_call_function_many_cond()
689 			csd_lock_wait(csd);  in smp_call_function_many_cond()