Lines Matching full:group
179 static void group_init(struct psi_group *group) in group_init() argument
183 group->enabled = true; in group_init()
185 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq); in group_init()
186 group->avg_last_update = sched_clock(); in group_init()
187 group->avg_next_update = group->avg_last_update + psi_period; in group_init()
188 mutex_init(&group->avgs_lock); in group_init()
191 INIT_LIST_HEAD(&group->avg_triggers); in group_init()
192 memset(group->avg_nr_triggers, 0, sizeof(group->avg_nr_triggers)); in group_init()
193 INIT_DELAYED_WORK(&group->avgs_work, psi_avgs_work); in group_init()
196 atomic_set(&group->rtpoll_scheduled, 0); in group_init()
197 mutex_init(&group->rtpoll_trigger_lock); in group_init()
198 INIT_LIST_HEAD(&group->rtpoll_triggers); in group_init()
199 group->rtpoll_min_period = U32_MAX; in group_init()
200 group->rtpoll_next_update = ULLONG_MAX; in group_init()
201 init_waitqueue_head(&group->rtpoll_wait); in group_init()
202 timer_setup(&group->rtpoll_timer, poll_timer_fn, 0); in group_init()
203 rcu_assign_pointer(group->rtpoll_task, NULL); in group_init()
245 static void get_recent_times(struct psi_group *group, int cpu, in get_recent_times() argument
249 struct psi_group_cpu *groupc = per_cpu_ptr(group->pcpu, cpu); in get_recent_times()
301 if (current_work() == &group->avgs_work.work) { in get_recent_times()
336 static void collect_percpu_times(struct psi_group *group, in collect_percpu_times() argument
359 get_recent_times(group, cpu, aggregator, times, in collect_percpu_times()
384 group->total[aggregator][s] += in collect_percpu_times()
437 static u64 update_triggers(struct psi_group *group, u64 now, bool *update_total, in update_triggers() argument
441 u64 *total = group->total[aggregator]; in update_triggers()
447 triggers = &group->avg_triggers; in update_triggers()
448 aggregator_total = group->avg_total; in update_triggers()
450 triggers = &group->rtpoll_triggers; in update_triggers()
451 aggregator_total = group->rtpoll_total; in update_triggers()
476 * remember to update group->polling_total[] once we've in update_triggers()
507 return now + group->rtpoll_min_period; in update_triggers()
510 static u64 update_averages(struct psi_group *group, u64 now) in update_averages() argument
518 expires = group->avg_next_update; in update_averages()
530 period = now - (group->avg_last_update + (missed_periods * psi_period)); in update_averages()
531 group->avg_last_update = now; in update_averages()
536 sample = group->total[PSI_AVGS][s] - group->avg_total[s]; in update_averages()
556 group->avg_total[s] += sample; in update_averages()
557 calc_avgs(group->avg[s], missed_periods, sample, period); in update_averages()
566 struct psi_group *group; in psi_avgs_work() local
572 group = container_of(dwork, struct psi_group, avgs_work); in psi_avgs_work()
574 mutex_lock(&group->avgs_lock); in psi_avgs_work()
578 collect_percpu_times(group, PSI_AVGS, &changed_states); in psi_avgs_work()
586 if (now >= group->avg_next_update) { in psi_avgs_work()
587 update_triggers(group, now, &update_total, PSI_AVGS); in psi_avgs_work()
588 group->avg_next_update = update_averages(group, now); in psi_avgs_work()
593 group->avg_next_update - now) + 1); in psi_avgs_work()
596 mutex_unlock(&group->avgs_lock); in psi_avgs_work()
599 static void init_rtpoll_triggers(struct psi_group *group, u64 now) in init_rtpoll_triggers() argument
603 list_for_each_entry(t, &group->rtpoll_triggers, node) in init_rtpoll_triggers()
605 group->total[PSI_POLL][t->state], 0); in init_rtpoll_triggers()
606 memcpy(group->rtpoll_total, group->total[PSI_POLL], in init_rtpoll_triggers()
607 sizeof(group->rtpoll_total)); in init_rtpoll_triggers()
608 group->rtpoll_next_update = now + group->rtpoll_min_period; in init_rtpoll_triggers()
612 static void psi_schedule_rtpoll_work(struct psi_group *group, unsigned long delay, in psi_schedule_rtpoll_work() argument
621 if (atomic_xchg(&group->rtpoll_scheduled, 1) && !force) in psi_schedule_rtpoll_work()
626 task = rcu_dereference(group->rtpoll_task); in psi_schedule_rtpoll_work()
632 mod_timer(&group->rtpoll_timer, jiffies + delay); in psi_schedule_rtpoll_work()
634 atomic_set(&group->rtpoll_scheduled, 0); in psi_schedule_rtpoll_work()
639 static void psi_rtpoll_work(struct psi_group *group) in psi_rtpoll_work() argument
646 mutex_lock(&group->rtpoll_trigger_lock); in psi_rtpoll_work()
650 if (now > group->rtpoll_until) { in psi_rtpoll_work()
660 atomic_set(&group->rtpoll_scheduled, 0); in psi_rtpoll_work()
687 collect_percpu_times(group, PSI_POLL, &changed_states); in psi_rtpoll_work()
689 if (changed_states & group->rtpoll_states) { in psi_rtpoll_work()
691 if (now > group->rtpoll_until) in psi_rtpoll_work()
692 init_rtpoll_triggers(group, now); in psi_rtpoll_work()
699 group->rtpoll_until = now + in psi_rtpoll_work()
700 group->rtpoll_min_period * UPDATES_PER_WINDOW; in psi_rtpoll_work()
703 if (now > group->rtpoll_until) { in psi_rtpoll_work()
704 group->rtpoll_next_update = ULLONG_MAX; in psi_rtpoll_work()
708 if (now >= group->rtpoll_next_update) { in psi_rtpoll_work()
709 group->rtpoll_next_update = update_triggers(group, now, &update_total, PSI_POLL); in psi_rtpoll_work()
711 memcpy(group->rtpoll_total, group->total[PSI_POLL], in psi_rtpoll_work()
712 sizeof(group->rtpoll_total)); in psi_rtpoll_work()
715 psi_schedule_rtpoll_work(group, in psi_rtpoll_work()
716 nsecs_to_jiffies(group->rtpoll_next_update - now) + 1, in psi_rtpoll_work()
720 mutex_unlock(&group->rtpoll_trigger_lock); in psi_rtpoll_work()
725 struct psi_group *group = (struct psi_group *)data; in psi_rtpoll_worker() local
730 wait_event_interruptible(group->rtpoll_wait, in psi_rtpoll_worker()
731 atomic_cmpxchg(&group->rtpoll_wakeup, 1, 0) || in psi_rtpoll_worker()
736 psi_rtpoll_work(group); in psi_rtpoll_worker()
743 struct psi_group *group = from_timer(group, t, rtpoll_timer); in poll_timer_fn() local
745 atomic_set(&group->rtpoll_wakeup, 1); in poll_timer_fn()
746 wake_up_interruptible(&group->rtpoll_wait); in poll_timer_fn()
778 static void psi_group_change(struct psi_group *group, int cpu, in psi_group_change() argument
787 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_group_change()
838 if (!group->enabled) { in psi_group_change()
840 * On the first group change after disabling PSI, conclude in psi_group_change()
877 if (state_mask & group->rtpoll_states) in psi_group_change()
878 psi_schedule_rtpoll_work(group, 1, false); in psi_group_change()
880 if (wake_clock && !delayed_work_pending(&group->avgs_work)) in psi_group_change()
881 schedule_delayed_work(&group->avgs_work, PSI_FREQ); in psi_group_change()
911 struct psi_group *group; in psi_task_change() local
921 group = task_psi_group(task); in psi_task_change()
923 psi_group_change(group, cpu, clear, set, now, true); in psi_task_change()
924 } while ((group = group->parent)); in psi_task_change()
930 struct psi_group *group, *common = NULL; in psi_task_switch() local
941 group = task_psi_group(next); in psi_task_switch()
943 if (per_cpu_ptr(group->pcpu, cpu)->state_mask & in psi_task_switch()
945 common = group; in psi_task_switch()
949 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true); in psi_task_switch()
950 } while ((group = group->parent)); in psi_task_switch()
983 group = task_psi_group(prev); in psi_task_switch()
985 if (group == common) in psi_task_switch()
987 psi_group_change(group, cpu, clear, set, now, wake_clock); in psi_task_switch()
988 } while ((group = group->parent)); in psi_task_switch()
998 for (; group; group = group->parent) in psi_task_switch()
999 psi_group_change(group, cpu, clear, set, now, wake_clock); in psi_task_switch()
1008 struct psi_group *group; in psi_account_irqtime() local
1017 group = task_psi_group(task); in psi_account_irqtime()
1019 if (!group->enabled) in psi_account_irqtime()
1022 groupc = per_cpu_ptr(group->pcpu, cpu); in psi_account_irqtime()
1031 if (group->rtpoll_states & (1 << PSI_IRQ_FULL)) in psi_account_irqtime()
1032 psi_schedule_rtpoll_work(group, 1, false); in psi_account_irqtime()
1033 } while ((group = group->parent)); in psi_account_irqtime()
1198 void psi_cgroup_restart(struct psi_group *group) in psi_cgroup_restart() argument
1217 if (!group->enabled) in psi_cgroup_restart()
1227 psi_group_change(group, cpu, 0, 0, now, true); in psi_cgroup_restart()
1233 int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) in psi_show() argument
1243 mutex_lock(&group->avgs_lock); in psi_show()
1245 collect_percpu_times(group, PSI_AVGS, NULL); in psi_show()
1246 if (now >= group->avg_next_update) in psi_show()
1247 group->avg_next_update = update_averages(group, now); in psi_show()
1248 mutex_unlock(&group->avgs_lock); in psi_show()
1260 if (!(group == &psi_system && res == PSI_CPU && full)) { in psi_show()
1262 avg[w] = group->avg[res * 2 + full][w]; in psi_show()
1263 total = div_u64(group->total[PSI_AVGS][res * 2 + full], in psi_show()
1278 struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, in psi_trigger_create() argument
1330 t->group = group; in psi_trigger_create()
1335 group->total[PSI_POLL][t->state], 0); in psi_trigger_create()
1346 mutex_lock(&group->rtpoll_trigger_lock); in psi_trigger_create()
1348 if (!rcu_access_pointer(group->rtpoll_task)) { in psi_trigger_create()
1351 task = kthread_create(psi_rtpoll_worker, group, "psimon"); in psi_trigger_create()
1354 mutex_unlock(&group->rtpoll_trigger_lock); in psi_trigger_create()
1357 atomic_set(&group->rtpoll_wakeup, 0); in psi_trigger_create()
1359 rcu_assign_pointer(group->rtpoll_task, task); in psi_trigger_create()
1362 list_add(&t->node, &group->rtpoll_triggers); in psi_trigger_create()
1363 group->rtpoll_min_period = min(group->rtpoll_min_period, in psi_trigger_create()
1365 group->rtpoll_nr_triggers[t->state]++; in psi_trigger_create()
1366 group->rtpoll_states |= (1 << t->state); in psi_trigger_create()
1368 mutex_unlock(&group->rtpoll_trigger_lock); in psi_trigger_create()
1370 mutex_lock(&group->avgs_lock); in psi_trigger_create()
1372 list_add(&t->node, &group->avg_triggers); in psi_trigger_create()
1373 group->avg_nr_triggers[t->state]++; in psi_trigger_create()
1375 mutex_unlock(&group->avgs_lock); in psi_trigger_create()
1382 struct psi_group *group; in psi_trigger_destroy() local
1392 group = t->group; in psi_trigger_destroy()
1404 mutex_lock(&group->avgs_lock); in psi_trigger_destroy()
1407 group->avg_nr_triggers[t->state]--; in psi_trigger_destroy()
1409 mutex_unlock(&group->avgs_lock); in psi_trigger_destroy()
1411 mutex_lock(&group->rtpoll_trigger_lock); in psi_trigger_destroy()
1417 group->rtpoll_nr_triggers[t->state]--; in psi_trigger_destroy()
1418 if (!group->rtpoll_nr_triggers[t->state]) in psi_trigger_destroy()
1419 group->rtpoll_states &= ~(1 << t->state); in psi_trigger_destroy()
1424 if (group->rtpoll_min_period == div_u64(t->win.size, UPDATES_PER_WINDOW)) { in psi_trigger_destroy()
1425 list_for_each_entry(tmp, &group->rtpoll_triggers, node) in psi_trigger_destroy()
1428 group->rtpoll_min_period = period; in psi_trigger_destroy()
1431 if (group->rtpoll_states == 0) { in psi_trigger_destroy()
1432 group->rtpoll_until = 0; in psi_trigger_destroy()
1434 group->rtpoll_task, in psi_trigger_destroy()
1435 lockdep_is_held(&group->rtpoll_trigger_lock)); in psi_trigger_destroy()
1436 rcu_assign_pointer(group->rtpoll_task, NULL); in psi_trigger_destroy()
1437 del_timer(&group->rtpoll_timer); in psi_trigger_destroy()
1440 mutex_unlock(&group->rtpoll_trigger_lock); in psi_trigger_destroy()
1457 * can no longer be found through group->rtpoll_task. in psi_trigger_destroy()
1460 atomic_set(&group->rtpoll_scheduled, 0); in psi_trigger_destroy()