280 	struct osnoise_variables *osn_var;  in osn_var_reset()  local
288 		osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in osn_var_reset()
289 		memset(osn_var, 0, sizeof(*osn_var));  in osn_var_reset()
371 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
379 		osn_var->softirq.arrival_time = 0;  in timerlat_softirq_exit()
380 		osn_var->softirq.delta_start = 0;  in timerlat_softirq_exit()
386 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
394 		osn_var->thread.delta_start = 0;  in timerlat_thread_exit()
395 		osn_var->thread.arrival_time = 0;  in timerlat_thread_exit()
406 static inline int timerlat_softirq_exit(struct osnoise_variables *osn_var)  in timerlat_softirq_exit()  argument
410 static inline int timerlat_thread_exit(struct osnoise_variables *osn_var)  in timerlat_thread_exit()  argument
720 cond_move_irq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_irq_delta_start()  argument
722 	if (osn_var->irq.delta_start)  in cond_move_irq_delta_start()
723 		osn_var->irq.delta_start += duration;  in cond_move_irq_delta_start()
736 cond_move_softirq_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_softirq_delta_start()  argument
738 	if (osn_var->softirq.delta_start)  in cond_move_softirq_delta_start()
739 		osn_var->softirq.delta_start += duration;  in cond_move_softirq_delta_start()
742 #define cond_move_softirq_delta_start(osn_var, duration) do {} while (0)  argument
754 cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration)  in cond_move_thread_delta_start()  argument
756 	if (osn_var->thread.delta_start)  in cond_move_thread_delta_start()
757 		osn_var->thread.delta_start += duration;  in cond_move_thread_delta_start()
788 get_int_safe_duration(struct osnoise_variables *osn_var, u64 *delta_start)  in get_int_safe_duration()  argument
794 		int_counter = local_read(&osn_var->int_counter);  in get_int_safe_duration()
803 	} while (int_counter != local_read(&osn_var->int_counter));  in get_int_safe_duration()
827 set_int_safe_time(struct osnoise_variables *osn_var, u64 *time)  in set_int_safe_time()  argument
832 		int_counter = local_read(&osn_var->int_counter);  in set_int_safe_time()
840 	} while (int_counter != local_read(&osn_var->int_counter));  in set_int_safe_time()
850 copy_int_safe_time(struct osnoise_variables *osn_var, u64 *dst, u64 *src)  in copy_int_safe_time()  argument
855 		int_counter = local_read(&osn_var->int_counter);  in copy_int_safe_time()
863 	} while (int_counter != local_read(&osn_var->int_counter));  in copy_int_safe_time()
879 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_osnoise_callback()  local
882 	if (!osn_var->sampling)  in trace_osnoise_callback()
891 			osn_var->nmi.delta_start = time_get();  in trace_osnoise_callback()
892 			local_inc(&osn_var->int_counter);  in trace_osnoise_callback()
894 			duration = time_get() - osn_var->nmi.delta_start;  in trace_osnoise_callback()
896 			trace_nmi_noise(osn_var->nmi.delta_start, duration);  in trace_osnoise_callback()
898 			cond_move_irq_delta_start(osn_var, duration);  in trace_osnoise_callback()
899 			cond_move_softirq_delta_start(osn_var, duration);  in trace_osnoise_callback()
900 			cond_move_thread_delta_start(osn_var, duration);  in trace_osnoise_callback()
905 		osn_var->nmi.count++;  in trace_osnoise_callback()
919 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_entry()  local
921 	if (!osn_var->sampling)  in osnoise_trace_irq_entry()
927 	osn_var->irq.arrival_time = time_get();  in osnoise_trace_irq_entry()
928 	set_int_safe_time(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_entry()
929 	osn_var->irq.count++;  in osnoise_trace_irq_entry()
931 	local_inc(&osn_var->int_counter);  in osnoise_trace_irq_entry()
942 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in osnoise_trace_irq_exit()  local
945 	if (!osn_var->sampling)  in osnoise_trace_irq_exit()
948 	duration = get_int_safe_duration(osn_var, &osn_var->irq.delta_start);  in osnoise_trace_irq_exit()
949 	trace_irq_noise(id, desc, osn_var->irq.arrival_time, duration);  in osnoise_trace_irq_exit()
950 	osn_var->irq.arrival_time = 0;  in osnoise_trace_irq_exit()
951 	cond_move_softirq_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
952 	cond_move_thread_delta_start(osn_var, duration);  in osnoise_trace_irq_exit()
1050 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_entry_callback()  local
1052 	if (!osn_var->sampling)  in trace_softirq_entry_callback()
1058 	osn_var->softirq.arrival_time = time_get();  in trace_softirq_entry_callback()
1059 	set_int_safe_time(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_entry_callback()
1060 	osn_var->softirq.count++;  in trace_softirq_entry_callback()
1062 	local_inc(&osn_var->int_counter);  in trace_softirq_entry_callback()
1073 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_softirq_exit_callback()  local
1076 	if (!osn_var->sampling)  in trace_softirq_exit_callback()
1080 		if (!timerlat_softirq_exit(osn_var))  in trace_softirq_exit_callback()
1083 	duration = get_int_safe_duration(osn_var, &osn_var->softirq.delta_start);  in trace_softirq_exit_callback()
1084 	trace_softirq_noise(vec_nr, osn_var->softirq.arrival_time, duration);  in trace_softirq_exit_callback()
1085 	cond_move_thread_delta_start(osn_var, duration);  in trace_softirq_exit_callback()
1086 	osn_var->softirq.arrival_time = 0;  in trace_softirq_exit_callback()
1146 thread_entry(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_entry()  argument
1148 	if (!osn_var->sampling)  in thread_entry()
1154 	osn_var->thread.arrival_time = time_get();  in thread_entry()
1156 	set_int_safe_time(osn_var, &osn_var->thread.delta_start);  in thread_entry()
1158 	osn_var->thread.count++;  in thread_entry()
1159 	local_inc(&osn_var->int_counter);  in thread_entry()
1168 thread_exit(struct osnoise_variables *osn_var, struct task_struct *t)  in thread_exit()  argument
1172 	if (!osn_var->sampling)  in thread_exit()
1176 		if (!timerlat_thread_exit(osn_var))  in thread_exit()
1179 	duration = get_int_safe_duration(osn_var, &osn_var->thread.delta_start);  in thread_exit()
1181 	trace_thread_noise(t, osn_var->thread.arrival_time, duration);  in thread_exit()
1183 	osn_var->thread.arrival_time = 0;  in thread_exit()
1221 	struct osnoise_variables *osn_var;  in trace_sched_migrate_callback()  local
1224 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in trace_sched_migrate_callback()
1225 	if (osn_var->pid == p->pid && dest_cpu != cpu) {  in trace_sched_migrate_callback()
1270 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in trace_sched_switch_callback()  local
1273 	if ((p->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1274 		thread_exit(osn_var, p);  in trace_sched_switch_callback()
1276 	if ((n->pid != osn_var->pid) || !workload)  in trace_sched_switch_callback()
1277 		thread_entry(osn_var, n);  in trace_sched_switch_callback()
1325 save_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in save_osn_sample_stats()  argument
1327 	s->nmi_count = osn_var->nmi.count;  in save_osn_sample_stats()
1328 	s->irq_count = osn_var->irq.count;  in save_osn_sample_stats()
1329 	s->softirq_count = osn_var->softirq.count;  in save_osn_sample_stats()
1330 	s->thread_count = osn_var->thread.count;  in save_osn_sample_stats()
1341 diff_osn_sample_stats(struct osnoise_variables *osn_var, struct osnoise_sample *s)  in diff_osn_sample_stats()  argument
1343 	s->nmi_count = osn_var->nmi.count - s->nmi_count;  in diff_osn_sample_stats()
1344 	s->irq_count = osn_var->irq.count - s->irq_count;  in diff_osn_sample_stats()
1345 	s->softirq_count = osn_var->softirq.count - s->softirq_count;  in diff_osn_sample_stats()
1346 	s->thread_count = osn_var->thread.count - s->thread_count;  in diff_osn_sample_stats()
1417 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in run_osnoise()  local
1439 	osn_var->pid = current->pid;  in run_osnoise()
1444 	save_osn_sample_stats(osn_var, &s);  in run_osnoise()
1463 	osn_var->sampling = true;  in run_osnoise()
1481 	last_int_count = set_int_safe_time(osn_var, &last_sample);  in run_osnoise()
1487 		int_count = set_int_safe_time(osn_var, &sample);  in run_osnoise()
1573 	osn_var->sampling = false;  in run_osnoise()
1598 	diff_osn_sample_stats(osn_var, &s);  in run_osnoise()
1729 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_irq()  local
1749 	osn_var->thread.arrival_time = time_get();  in timerlat_irq()
1772 	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && osn_var->softirq.delta_start) {  in timerlat_irq()
1773 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1774 				   &osn_var->softirq.delta_start);  in timerlat_irq()
1776 		copy_int_safe_time(osn_var, &osn_var->softirq.delta_start,  in timerlat_irq()
1777 				    &osn_var->irq.delta_start);  in timerlat_irq()
1779 		copy_int_safe_time(osn_var, &osn_var->thread.delta_start,  in timerlat_irq()
1780 				    &osn_var->irq.delta_start);  in timerlat_irq()
1863 	struct osnoise_variables *osn_var = this_cpu_osn_var();  in timerlat_main()  local
1894 	osn_var->pid = current->pid;  in timerlat_main()
1902 	osn_var->sampling = 1;  in timerlat_main()
2388 	struct osnoise_variables *osn_var;  in timerlat_fd_open()  local
2405 	osn_var = this_cpu_osn_var();  in timerlat_fd_open()
2410 	if (osn_var->pid) {  in timerlat_fd_open()
2436 	osn_var->kthread = current;  in timerlat_fd_open()
2437 	osn_var->pid = current->pid;  in timerlat_fd_open()
2465 	struct osnoise_variables *osn_var;  in timerlat_fd_read()  local
2493 	osn_var = this_cpu_osn_var();  in timerlat_fd_read()
2503 	if (likely(osn_var->sampling)) {  in timerlat_fd_read()
2535 		osn_var->sampling = 1;  in timerlat_fd_read()
2572 	struct osnoise_variables *osn_var;  in timerlat_fd_release()  local
2579 	osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu);  in timerlat_fd_release()
2585 	osn_var->sampling = 0;  in timerlat_fd_release()
2586 	osn_var->pid = 0;  in timerlat_fd_release()
2591 	if (osn_var->kthread) {  in timerlat_fd_release()
2592 		put_task_struct(osn_var->kthread);  in timerlat_fd_release()
2593 		osn_var->kthread = NULL;  in timerlat_fd_release()