245 	u64 now, state_start;  in get_recent_times()  local
255 		now = cpu_clock(cpu);  in get_recent_times()
274 			times[s] += now - state_start;  in get_recent_times()
360 static u64 update_averages(struct psi_group *group, u64 now)  in update_averages()  argument
369 	if (now - expires >= psi_period)  in update_averages()
370 		missed_periods = div_u64(now - expires, psi_period);  in update_averages()
380 	period = now - (group->avg_last_update + (missed_periods * psi_period));  in update_averages()
381 	group->avg_last_update = now;  in update_averages()
419 	u64 now;  in psi_avgs_work()  local
426 	now = sched_clock();  in psi_avgs_work()
437 	if (now >= group->avg_next_update)  in psi_avgs_work()
438 		group->avg_next_update = update_averages(group, now);  in psi_avgs_work()
442 				group->avg_next_update - now) + 1);  in psi_avgs_work()
449 static void window_reset(struct psi_window *win, u64 now, u64 value,  in window_reset()  argument
452 	win->start_time = now;  in window_reset()
468 static u64 window_update(struct psi_window *win, u64 now, u64 value)  in window_update()  argument
473 	elapsed = now - win->start_time;  in window_update()
483 		window_reset(win, now, value, growth);  in window_update()
494 static void init_triggers(struct psi_group *group, u64 now)  in init_triggers()  argument
499 		window_reset(&t->win, now,  in init_triggers()
503 	group->polling_next_update = now + group->poll_min_period;  in init_triggers()
506 static u64 update_triggers(struct psi_group *group, u64 now)  in update_triggers()  argument
541 			growth = window_update(&t->win, now, total[t->state]);  in update_triggers()
548 		if (now < t->last_event_time + t->win.size)  in update_triggers()
554 		t->last_event_time = now;  in update_triggers()
563 	return now + group->poll_min_period;  in update_triggers()
596 	u64 now;  in psi_poll_work()  local
600 	now = sched_clock();  in psi_poll_work()
606 		if (now > group->polling_until)  in psi_poll_work()
607 			init_triggers(group, now);  in psi_poll_work()
614 		group->polling_until = now +  in psi_poll_work()
618 	if (now > group->polling_until) {  in psi_poll_work()
623 	if (now >= group->polling_next_update)  in psi_poll_work()
624 		group->polling_next_update = update_triggers(group, now);  in psi_poll_work()
627 		nsecs_to_jiffies(group->polling_next_update - now) + 1);  in psi_poll_work()
659 static void record_times(struct psi_group_cpu *groupc, u64 now)  in record_times()  argument
663 	delta = now - groupc->state_start;  in record_times()
664 	groupc->state_start = now;  in record_times()
689 			     unsigned int clear, unsigned int set, u64 now,  in psi_group_change()  argument
757 			record_times(groupc, now);  in psi_group_change()
781 	record_times(groupc, now);  in psi_group_change()
822 	u64 now;  in psi_task_change()  local
829 	now = cpu_clock(cpu);  in psi_task_change()
833 		psi_group_change(group, cpu, clear, set, now, true);  in psi_task_change()
842 	u64 now = cpu_clock(cpu);  in psi_task_switch()  local
859 			psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);  in psi_task_switch()
897 			psi_group_change(group, cpu, clear, set, now, wake_clock);  in psi_task_switch()
909 				psi_group_change(group, cpu, clear, set, now, wake_clock);  in psi_task_switch()
920 	u64 now;  in psi_account_irqtime()  local
925 	now = cpu_clock(cpu);  in psi_account_irqtime()
936 		record_times(groupc, now);  in psi_account_irqtime()
1036 	/* All triggers must be removed by now */  in psi_cgroup_free()
1124 	 * and restart groupc->state_start from now, use .clear = .set = 0  in psi_cgroup_restart()
1133 		u64 now;  in psi_cgroup_restart()  local
1136 		now = cpu_clock(cpu);  in psi_cgroup_restart()
1137 		psi_group_change(group, cpu, 0, 0, now, true);  in psi_cgroup_restart()
1147 	u64 now;  in psi_show()  local
1154 	now = sched_clock();  in psi_show()
1156 	if (now >= group->avg_next_update)  in psi_show()
1157 		group->avg_next_update = update_averages(group, now);  in psi_show()