Lines Matching +full:reserved +full:- +full:ipi +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0
6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
75 struct task_struct *p = tfc->p; in remote_function()
78 /* -EAGAIN */ in remote_function()
87 tfc->ret = -ESRCH; /* No such (running) process */ in remote_function()
92 tfc->ret = tfc->func(tfc->info); in remote_function()
96 * task_function_call - call a function on the cpu on which a task runs
106 * returns @func return value or -ESRCH or -ENXIO when the process isn't running
115 .ret = -EAGAIN, in task_function_call()
125 if (ret != -EAGAIN) in task_function_call()
135 * cpu_function_call - call a function on the cpu
142 * returns: @func return value or -ENXIO when the cpu is offline
150 .ret = -ENXIO, /* No such CPU */ in cpu_function_call()
161 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); in __get_cpu_context()
167 raw_spin_lock(&cpuctx->ctx.lock); in perf_ctx_lock()
169 raw_spin_lock(&ctx->lock); in perf_ctx_lock()
176 raw_spin_unlock(&ctx->lock); in perf_ctx_unlock()
177 raw_spin_unlock(&cpuctx->ctx.lock); in perf_ctx_unlock()
180 #define TASK_TOMBSTONE ((void *)-1L)
184 return READ_ONCE(event->owner) == TASK_TOMBSTONE; in is_kernel_event()
190 * When !ctx->nr_events a task context will not be scheduled. This means
196 * - removing the last event from a task ctx; this is relatively straight
199 * - adding the first event to a task ctx; this is tricky because we cannot
200 * rely on ctx->is_active and therefore cannot use event_function_call().
203 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set.
218 struct perf_event *event = efs->event; in event_function()
219 struct perf_event_context *ctx = event->ctx; in event_function()
221 struct perf_event_context *task_ctx = cpuctx->task_ctx; in event_function()
228 * Since we do the IPI call without holding ctx->lock things can have in event_function()
231 if (ctx->task) { in event_function()
232 if (ctx->task != current) { in event_function()
233 ret = -ESRCH; in event_function()
241 * above ctx->task != current test), therefore we must have in event_function()
242 * ctx->is_active here. in event_function()
244 WARN_ON_ONCE(!ctx->is_active); in event_function()
246 * And since we have ctx->is_active, cpuctx->task_ctx must in event_function()
251 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function()
254 efs->func(event, cpuctx, ctx, efs->data); in event_function()
263 struct perf_event_context *ctx = event->ctx; in event_function_call()
264 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ in event_function_call()
271 if (!event->parent) { in event_function_call()
274 * stabilize the event->ctx relation. See in event_function_call()
277 lockdep_assert_held(&ctx->mutex); in event_function_call()
281 cpu_function_call(event->cpu, event_function, &efs); in event_function_call()
292 raw_spin_lock_irq(&ctx->lock); in event_function_call()
297 task = ctx->task; in event_function_call()
299 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
302 if (ctx->is_active) { in event_function_call()
303 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
307 raw_spin_unlock_irq(&ctx->lock); in event_function_call()
316 struct perf_event_context *ctx = event->ctx; in event_function_local()
318 struct task_struct *task = READ_ONCE(ctx->task); in event_function_local()
332 task = ctx->task; in event_function_local()
339 * otherwise we're screwed, since we cannot IPI to somewhere in event_function_local()
342 if (ctx->is_active) { in event_function_local()
346 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) in event_function_local()
350 WARN_ON_ONCE(&cpuctx->ctx != ctx); in event_function_local()
381 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
414 * -1 - not paranoid at all
415 * 0 - disallow raw tracepoint access for unpriv
416 * 1 - disallow cpu events for unpriv
417 * 2 - disallow kernel profiling for unpriv
462 return -EINVAL; in perf_proc_update_handler()
532 running_len -= running_len/NR_ACCUMULATED_SAMPLES; in perf_sample_event_took()
592 return event->clock(); in perf_event_clock()
598 * The basic idea is to use event->state to determine which (if any) time
620 struct perf_event *leader = event->group_leader; in __perf_effective_state()
622 if (leader->state <= PERF_EVENT_STATE_OFF) in __perf_effective_state()
623 return leader->state; in __perf_effective_state()
625 return event->state; in __perf_effective_state()
632 u64 delta = now - event->tstamp; in __perf_update_times()
634 *enabled = event->total_time_enabled; in __perf_update_times()
638 *running = event->total_time_running; in __perf_update_times()
647 __perf_update_times(event, now, &event->total_time_enabled, in perf_event_update_time()
648 &event->total_time_running); in perf_event_update_time()
649 event->tstamp = now; in perf_event_update_time()
663 if (event->state == state) in perf_event_set_state()
671 if ((event->state < 0) ^ (state < 0)) in perf_event_set_state()
674 WRITE_ONCE(event->state, state); in perf_event_set_state()
678 * UP store-release, load-acquire
699 struct perf_event_context *ctx = event->ctx; in perf_cgroup_match()
703 if (!event->cgrp) in perf_cgroup_match()
707 if (!cpuctx->cgrp) in perf_cgroup_match()
716 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, in perf_cgroup_match()
717 event->cgrp->css.cgroup); in perf_cgroup_match()
722 css_put(&event->cgrp->css); in perf_detach_cgroup()
723 event->cgrp = NULL; in perf_detach_cgroup()
728 return event->cgrp != NULL; in is_cgroup_event()
735 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time()
736 return t->time; in perf_cgroup_event_time()
743 t = per_cpu_ptr(event->cgrp->info, event->cpu); in perf_cgroup_event_time_now()
744 if (!__load_acquire(&t->active)) in perf_cgroup_event_time_now()
745 return t->time; in perf_cgroup_event_time_now()
746 now += READ_ONCE(t->timeoffset); in perf_cgroup_event_time_now()
753 info->time += now - info->timestamp; in __update_cgrp_time()
754 info->timestamp = now; in __update_cgrp_time()
758 WRITE_ONCE(info->timeoffset, info->time - info->timestamp); in __update_cgrp_time()
763 struct perf_cgroup *cgrp = cpuctx->cgrp; in update_cgrp_time_from_cpuctx()
770 for (css = &cgrp->css; css; css = css->parent) { in update_cgrp_time_from_cpuctx()
772 info = this_cpu_ptr(cgrp->info); in update_cgrp_time_from_cpuctx()
776 __store_release(&info->active, 0); in update_cgrp_time_from_cpuctx()
792 info = this_cpu_ptr(event->cgrp->info); in update_cgrp_time_from_event()
796 if (info->active) in update_cgrp_time_from_event()
803 struct perf_event_context *ctx = &cpuctx->ctx; in perf_cgroup_set_timestamp()
804 struct perf_cgroup *cgrp = cpuctx->cgrp; in perf_cgroup_set_timestamp()
809 * ctx->lock held by caller in perf_cgroup_set_timestamp()
816 WARN_ON_ONCE(!ctx->nr_cgroups); in perf_cgroup_set_timestamp()
818 for (css = &cgrp->css; css; css = css->parent) { in perf_cgroup_set_timestamp()
820 info = this_cpu_ptr(cgrp->info); in perf_cgroup_set_timestamp()
821 __update_cgrp_time(info, ctx->timestamp, false); in perf_cgroup_set_timestamp()
822 __store_release(&info->active, 1); in perf_cgroup_set_timestamp()
848 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); in perf_cgroup_switch()
849 if (READ_ONCE(cpuctx->cgrp) == cgrp) in perf_cgroup_switch()
852 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
853 perf_pmu_disable(cpuctx->ctx.pmu); in perf_cgroup_switch()
861 cpuctx->cgrp = cgrp; in perf_cgroup_switch()
869 perf_pmu_enable(cpuctx->ctx.pmu); in perf_cgroup_switch()
870 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_cgroup_switch()
887 for (heap_size = 1; css; css = css->parent) in perf_cgroup_ensure_storage()
891 cpuctx = per_cpu_ptr(event->pmu->pmu_cpu_context, cpu); in perf_cgroup_ensure_storage()
892 if (heap_size <= cpuctx->heap_size) in perf_cgroup_ensure_storage()
898 ret = -ENOMEM; in perf_cgroup_ensure_storage()
902 raw_spin_lock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
903 if (cpuctx->heap_size < heap_size) { in perf_cgroup_ensure_storage()
904 swap(cpuctx->heap, storage); in perf_cgroup_ensure_storage()
905 if (storage == cpuctx->heap_default) in perf_cgroup_ensure_storage()
907 cpuctx->heap_size = heap_size; in perf_cgroup_ensure_storage()
909 raw_spin_unlock_irq(&cpuctx->ctx.lock); in perf_cgroup_ensure_storage()
927 return -EBADF; in perf_cgroup_connect()
929 css = css_tryget_online_from_dir(f.file->f_path.dentry, in perf_cgroup_connect()
941 event->cgrp = cgrp; in perf_cgroup_connect()
948 if (group_leader && group_leader->cgrp != cgrp) { in perf_cgroup_connect()
950 ret = -EINVAL; in perf_cgroup_connect()
966 * Because cgroup events are always per-cpu events, in perf_cgroup_event_enable()
967 * @ctx == &cpuctx->ctx. in perf_cgroup_event_enable()
971 if (ctx->nr_cgroups++) in perf_cgroup_event_enable()
974 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); in perf_cgroup_event_enable()
975 list_add(&cpuctx->cgrp_cpuctx_entry, in perf_cgroup_event_enable()
976 per_cpu_ptr(&cgrp_cpuctx_list, event->cpu)); in perf_cgroup_event_enable()
988 * Because cgroup events are always per-cpu events, in perf_cgroup_event_disable()
989 * @ctx == &cpuctx->ctx. in perf_cgroup_event_disable()
993 if (--ctx->nr_cgroups) in perf_cgroup_event_disable()
996 cpuctx->cgrp = NULL; in perf_cgroup_event_disable()
997 list_del(&cpuctx->cgrp_cpuctx_entry); in perf_cgroup_event_disable()
1029 return -EINVAL; in perf_cgroup_connect()
1080 raw_spin_lock(&cpuctx->hrtimer_lock); in perf_mux_hrtimer_handler()
1082 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); in perf_mux_hrtimer_handler()
1084 cpuctx->hrtimer_active = 0; in perf_mux_hrtimer_handler()
1085 raw_spin_unlock(&cpuctx->hrtimer_lock); in perf_mux_hrtimer_handler()
1092 struct hrtimer *timer = &cpuctx->hrtimer; in __perf_mux_hrtimer_init()
1093 struct pmu *pmu = cpuctx->ctx.pmu; in __perf_mux_hrtimer_init()
1097 if (pmu->task_ctx_nr == perf_sw_context) in __perf_mux_hrtimer_init()
1104 interval = pmu->hrtimer_interval_ms; in __perf_mux_hrtimer_init()
1106 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; in __perf_mux_hrtimer_init()
1108 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); in __perf_mux_hrtimer_init()
1110 raw_spin_lock_init(&cpuctx->hrtimer_lock); in __perf_mux_hrtimer_init()
1112 timer->function = perf_mux_hrtimer_handler; in __perf_mux_hrtimer_init()
1117 struct hrtimer *timer = &cpuctx->hrtimer; in perf_mux_hrtimer_restart()
1118 struct pmu *pmu = cpuctx->ctx.pmu; in perf_mux_hrtimer_restart()
1122 if (pmu->task_ctx_nr == perf_sw_context) in perf_mux_hrtimer_restart()
1125 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1126 if (!cpuctx->hrtimer_active) { in perf_mux_hrtimer_restart()
1127 cpuctx->hrtimer_active = 1; in perf_mux_hrtimer_restart()
1128 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); in perf_mux_hrtimer_restart()
1131 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); in perf_mux_hrtimer_restart()
1138 int *count = this_cpu_ptr(pmu->pmu_disable_count); in perf_pmu_disable()
1140 pmu->pmu_disable(pmu); in perf_pmu_disable()
1145 int *count = this_cpu_ptr(pmu->pmu_disable_count); in perf_pmu_enable()
1146 if (!--(*count)) in perf_pmu_enable()
1147 pmu->pmu_enable(pmu); in perf_pmu_enable()
1164 WARN_ON(!list_empty(&ctx->active_ctx_list)); in perf_event_ctx_activate()
1166 list_add(&ctx->active_ctx_list, head); in perf_event_ctx_activate()
1173 WARN_ON(list_empty(&ctx->active_ctx_list)); in perf_event_ctx_deactivate()
1175 list_del_init(&ctx->active_ctx_list); in perf_event_ctx_deactivate()
1180 refcount_inc(&ctx->refcount); in get_ctx()
1185 if (pmu->task_ctx_cache) in alloc_task_ctx_data()
1186 return kmem_cache_zalloc(pmu->task_ctx_cache, GFP_KERNEL); in alloc_task_ctx_data()
1193 if (pmu->task_ctx_cache && task_ctx_data) in free_task_ctx_data()
1194 kmem_cache_free(pmu->task_ctx_cache, task_ctx_data); in free_task_ctx_data()
1202 free_task_ctx_data(ctx->pmu, ctx->task_ctx_data); in free_ctx()
1208 if (refcount_dec_and_test(&ctx->refcount)) { in put_ctx()
1209 if (ctx->parent_ctx) in put_ctx()
1210 put_ctx(ctx->parent_ctx); in put_ctx()
1211 if (ctx->task && ctx->task != TASK_TOMBSTONE) in put_ctx()
1212 put_task_struct(ctx->task); in put_ctx()
1213 call_rcu(&ctx->rcu_head, free_ctx); in put_ctx()
1227 * - perf_event_exit_task_context() [ child , 0 ]
1231 * - perf_event_init_context() [ parent, 0 ]
1239 * While it appears there is an obvious deadlock here -- the parent and child
1241 * life-time rules separate them. That is an exiting task cannot fork, and a
1244 * But remember that these are parent<->child context relations, and
1250 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only
1261 * means that only external vectors (perf_fops, prctl) can perturb the event
1265 * However; because event->ctx can change while we're waiting to acquire
1266 * ctx->mutex we must be careful and use the below perf_event_ctx_lock()
1281 * cpuctx->mutex / perf_event_context::mutex
1290 ctx = READ_ONCE(event->ctx); in perf_event_ctx_lock_nested()
1291 if (!refcount_inc_not_zero(&ctx->refcount)) { in perf_event_ctx_lock_nested()
1297 mutex_lock_nested(&ctx->mutex, nesting); in perf_event_ctx_lock_nested()
1298 if (event->ctx != ctx) { in perf_event_ctx_lock_nested()
1299 mutex_unlock(&ctx->mutex); in perf_event_ctx_lock_nested()
1316 mutex_unlock(&ctx->mutex); in perf_event_ctx_unlock()
1321 * This must be done under the ctx->lock, such as to serialize against
1323 * calling scheduler related locks and ctx->lock nests inside those.
1328 struct perf_event_context *parent_ctx = ctx->parent_ctx; in unclone_ctx()
1330 lockdep_assert_held(&ctx->lock); in unclone_ctx()
1333 ctx->parent_ctx = NULL; in unclone_ctx()
1334 ctx->generation++; in unclone_ctx()
1346 if (event->parent) in perf_event_pid_type()
1347 event = event->parent; in perf_event_pid_type()
1349 nr = __task_pid_nr_ns(p, type, event->ns); in perf_event_pid_type()
1350 /* avoid -1 if it is idle thread or runs in another ns */ in perf_event_pid_type()
1352 nr = -1; in perf_event_pid_type()
1372 u64 id = event->id; in primary_event_id()
1374 if (event->parent) in primary_event_id()
1375 id = event->parent->id; in primary_event_id()
1395 * part of the read side critical section was irqs-enabled -- see in perf_lock_task_context()
1398 * Since ctx->lock nests under rq->lock we must ensure the entire read in perf_lock_task_context()
1403 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); in perf_lock_task_context()
1415 raw_spin_lock(&ctx->lock); in perf_lock_task_context()
1416 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { in perf_lock_task_context()
1417 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1423 if (ctx->task == TASK_TOMBSTONE || in perf_lock_task_context()
1424 !refcount_inc_not_zero(&ctx->refcount)) { in perf_lock_task_context()
1425 raw_spin_unlock(&ctx->lock); in perf_lock_task_context()
1428 WARN_ON_ONCE(ctx->task != task); in perf_lock_task_context()
1450 ++ctx->pin_count; in perf_pin_task_context()
1451 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_pin_task_context()
1460 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_unpin_context()
1461 --ctx->pin_count; in perf_unpin_context()
1462 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_unpin_context()
1472 lockdep_assert_held(&ctx->lock); in __update_context_time()
1475 ctx->time += now - ctx->timestamp; in __update_context_time()
1476 ctx->timestamp = now; in __update_context_time()
1479 * The above: time' = time + (now - timestamp), can be re-arranged in __update_context_time()
1480 * into: time` = now + (time - timestamp), which gives a single value in __update_context_time()
1484 * it's (obviously) not possible to acquire ctx->lock in order to read in __update_context_time()
1487 WRITE_ONCE(ctx->timeoffset, ctx->time - ctx->timestamp); in __update_context_time()
1497 struct perf_event_context *ctx = event->ctx; in perf_event_time()
1505 return ctx->time; in perf_event_time()
1510 struct perf_event_context *ctx = event->ctx; in perf_event_time_now()
1518 if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) in perf_event_time_now()
1519 return ctx->time; in perf_event_time_now()
1521 now += READ_ONCE(ctx->timeoffset); in perf_event_time_now()
1527 struct perf_event_context *ctx = event->ctx; in get_event_type()
1530 lockdep_assert_held(&ctx->lock); in get_event_type()
1536 if (event->group_leader != event) in get_event_type()
1537 event = event->group_leader; in get_event_type()
1539 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; in get_event_type()
1540 if (!ctx->task) in get_event_type()
1551 RB_CLEAR_NODE(&event->group_node); in init_event_group()
1552 event->group_index = 0; in init_event_group()
1562 if (event->attr.pinned) in get_event_groups()
1563 return &ctx->pinned_groups; in get_event_groups()
1565 return &ctx->flexible_groups; in get_event_groups()
1573 groups->tree = RB_ROOT; in perf_event_groups_init()
1574 groups->index = 0; in perf_event_groups_init()
1582 if (event->cgrp) in event_cgroup()
1583 cgroup = event->cgrp->css.cgroup; in event_cgroup()
1599 if (left_cpu < right->cpu) in perf_event_groups_cmp()
1600 return -1; in perf_event_groups_cmp()
1601 if (left_cpu > right->cpu) in perf_event_groups_cmp()
1614 return -1; in perf_event_groups_cmp()
1625 return -1; in perf_event_groups_cmp()
1632 if (left_group_index < right->group_index) in perf_event_groups_cmp()
1633 return -1; in perf_event_groups_cmp()
1634 if (left_group_index > right->group_index) in perf_event_groups_cmp()
1646 return perf_event_groups_cmp(e->cpu, event_cgroup(e), e->group_index, in __group_less()
1661 return perf_event_groups_cmp(a->cpu, a->cgroup, b->group_index, b); in __group_cmp()
1665 * Insert @event into @groups' tree; using {@event->cpu, ++@groups->index} for
1673 event->group_index = ++groups->index; in perf_event_groups_insert()
1675 rb_add(&event->group_node, &groups->tree, __group_less); in perf_event_groups_insert()
1697 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || in perf_event_groups_delete()
1698 RB_EMPTY_ROOT(&groups->tree)); in perf_event_groups_delete()
1700 rb_erase(&event->group_node, &groups->tree); in perf_event_groups_delete()
1729 node = rb_find_first(&key, &groups->tree, __group_cmp); in perf_event_groups_first()
1743 .cpu = event->cpu, in perf_event_groups_next()
1748 next = rb_next_match(&key, &event->group_node, __group_cmp); in perf_event_groups_next()
1759 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \
1761 event = rb_entry_safe(rb_next(&event->group_node), \
1766 * Must be called with ctx->mutex and ctx->lock held.
1771 lockdep_assert_held(&ctx->lock); in list_add_event()
1773 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); in list_add_event()
1774 event->attach_state |= PERF_ATTACH_CONTEXT; in list_add_event()
1776 event->tstamp = perf_event_time(event); in list_add_event()
1783 if (event->group_leader == event) { in list_add_event()
1784 event->group_caps = event->event_caps; in list_add_event()
1788 list_add_rcu(&event->event_entry, &ctx->event_list); in list_add_event()
1789 ctx->nr_events++; in list_add_event()
1790 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_add_event()
1791 ctx->nr_user++; in list_add_event()
1792 if (event->attr.inherit_stat) in list_add_event()
1793 ctx->nr_stat++; in list_add_event()
1795 if (event->state > PERF_EVENT_STATE_OFF) in list_add_event()
1798 ctx->generation++; in list_add_event()
1806 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : in perf_event__state_init()
1816 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) in __perf_event_read_size()
1819 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) in __perf_event_read_size()
1822 if (event->attr.read_format & PERF_FORMAT_ID) in __perf_event_read_size()
1825 if (event->attr.read_format & PERF_FORMAT_LOST) in __perf_event_read_size()
1828 if (event->attr.read_format & PERF_FORMAT_GROUP) { in __perf_event_read_size()
1834 event->read_size = size; in __perf_event_read_size()
1843 size += sizeof(data->ip); in __perf_event_header_size()
1846 size += sizeof(data->addr); in __perf_event_header_size()
1849 size += sizeof(data->period); in __perf_event_header_size()
1852 size += sizeof(data->weight.full); in __perf_event_header_size()
1855 size += event->read_size; in __perf_event_header_size()
1858 size += sizeof(data->data_src.val); in __perf_event_header_size()
1861 size += sizeof(data->txn); in __perf_event_header_size()
1864 size += sizeof(data->phys_addr); in __perf_event_header_size()
1867 size += sizeof(data->cgroup); in __perf_event_header_size()
1870 size += sizeof(data->data_page_size); in __perf_event_header_size()
1873 size += sizeof(data->code_page_size); in __perf_event_header_size()
1875 event->header_size = size; in __perf_event_header_size()
1885 event->group_leader->nr_siblings); in perf_event__header_size()
1886 __perf_event_header_size(event, event->attr.sample_type); in perf_event__header_size()
1892 u64 sample_type = event->attr.sample_type; in perf_event__id_header_size()
1896 size += sizeof(data->tid_entry); in perf_event__id_header_size()
1899 size += sizeof(data->time); in perf_event__id_header_size()
1902 size += sizeof(data->id); in perf_event__id_header_size()
1905 size += sizeof(data->id); in perf_event__id_header_size()
1908 size += sizeof(data->stream_id); in perf_event__id_header_size()
1911 size += sizeof(data->cpu_entry); in perf_event__id_header_size()
1913 event->id_header_size = size; in perf_event__id_header_size()
1919 * The values computed here will be over-written when we actually in perf_event_validate_size()
1922 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); in perf_event_validate_size()
1923 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); in perf_event_validate_size()
1930 if (event->read_size + event->header_size + in perf_event_validate_size()
1931 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) in perf_event_validate_size()
1939 struct perf_event *group_leader = event->group_leader, *pos; in perf_group_attach()
1941 lockdep_assert_held(&event->ctx->lock); in perf_group_attach()
1946 if (event->attach_state & PERF_ATTACH_GROUP) in perf_group_attach()
1949 event->attach_state |= PERF_ATTACH_GROUP; in perf_group_attach()
1954 WARN_ON_ONCE(group_leader->ctx != event->ctx); in perf_group_attach()
1956 group_leader->group_caps &= event->event_caps; in perf_group_attach()
1958 list_add_tail(&event->sibling_list, &group_leader->sibling_list); in perf_group_attach()
1959 group_leader->nr_siblings++; in perf_group_attach()
1969 * Must be called with ctx->mutex and ctx->lock held.
1974 WARN_ON_ONCE(event->ctx != ctx); in list_del_event()
1975 lockdep_assert_held(&ctx->lock); in list_del_event()
1978 * We can have double detach due to exit/hot-unplug + close. in list_del_event()
1980 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) in list_del_event()
1983 event->attach_state &= ~PERF_ATTACH_CONTEXT; in list_del_event()
1985 ctx->nr_events--; in list_del_event()
1986 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) in list_del_event()
1987 ctx->nr_user--; in list_del_event()
1988 if (event->attr.inherit_stat) in list_del_event()
1989 ctx->nr_stat--; in list_del_event()
1991 list_del_rcu(&event->event_entry); in list_del_event()
1993 if (event->group_leader == event) in list_del_event()
2000 * of error state is by explicit re-enabling in list_del_event()
2003 if (event->state > PERF_EVENT_STATE_OFF) { in list_del_event()
2008 ctx->generation++; in list_del_event()
2017 if (!event->pmu->aux_output_match) in perf_aux_output_match()
2020 return event->pmu->aux_output_match(aux_event); in perf_aux_output_match()
2030 struct perf_event_context *ctx = event->ctx; in perf_put_aux_event()
2037 if (event->aux_event) { in perf_put_aux_event()
2038 iter = event->aux_event; in perf_put_aux_event()
2039 event->aux_event = NULL; in perf_put_aux_event()
2048 for_each_sibling_event(iter, event->group_leader) { in perf_put_aux_event()
2049 if (iter->aux_event != event) in perf_put_aux_event()
2052 iter->aux_event = NULL; in perf_put_aux_event()
2067 return !!event->attr.aux_output || !!event->attr.aux_sample_size; in perf_need_aux_event()
2085 if (event->attr.aux_output && event->attr.aux_sample_size) in perf_get_aux_event()
2088 if (event->attr.aux_output && in perf_get_aux_event()
2092 if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux) in perf_get_aux_event()
2095 if (!atomic_long_inc_not_zero(&group_leader->refcount)) in perf_get_aux_event()
2104 event->aux_event = group_leader; in perf_get_aux_event()
2111 struct perf_event_context *ctx = event->ctx; in get_event_list()
2112 return event->attr.pinned ? &ctx->pinned_active : &ctx->flexible_active; in get_event_list()
2123 struct perf_event_context *ctx = event->ctx; in perf_remove_sibling_event()
2132 struct perf_event *leader = event->group_leader; in perf_group_detach()
2134 struct perf_event_context *ctx = event->ctx; in perf_group_detach()
2136 lockdep_assert_held(&ctx->lock); in perf_group_detach()
2139 * We can have double detach due to exit/hot-unplug + close. in perf_group_detach()
2141 if (!(event->attach_state & PERF_ATTACH_GROUP)) in perf_group_detach()
2144 event->attach_state &= ~PERF_ATTACH_GROUP; in perf_group_detach()
2152 list_del_init(&event->sibling_list); in perf_group_detach()
2153 event->group_leader->nr_siblings--; in perf_group_detach()
2162 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { in perf_group_detach()
2164 if (sibling->event_caps & PERF_EV_CAP_SIBLING) in perf_group_detach()
2167 sibling->group_leader = sibling; in perf_group_detach()
2168 list_del_init(&sibling->sibling_list); in perf_group_detach()
2171 sibling->group_caps = event->group_caps; in perf_group_detach()
2173 if (!RB_EMPTY_NODE(&event->group_node)) { in perf_group_detach()
2174 add_event_to_groups(sibling, event->ctx); in perf_group_detach()
2176 if (sibling->state == PERF_EVENT_STATE_ACTIVE) in perf_group_detach()
2177 list_add_tail(&sibling->active_list, get_event_list(sibling)); in perf_group_detach()
2180 WARN_ON_ONCE(sibling->ctx != event->ctx); in perf_group_detach()
2194 struct perf_event *parent_event = event->parent; in perf_child_detach()
2196 if (!(event->attach_state & PERF_ATTACH_CHILD)) in perf_child_detach()
2199 event->attach_state &= ~PERF_ATTACH_CHILD; in perf_child_detach()
2204 lockdep_assert_held(&parent_event->child_mutex); in perf_child_detach()
2207 list_del_init(&event->child_list); in perf_child_detach()
2212 return event->state == PERF_EVENT_STATE_DEAD; in is_orphaned_event()
2217 struct pmu *pmu = event->pmu; in __pmu_filter_match()
2218 return pmu->filter_match ? pmu->filter_match(event) : 1; in __pmu_filter_match()
2223 * PMU-specific filtering. An event group can consist of HW and SW events,
2251 return (event->cpu == -1 || event->cpu == smp_processor_id()) && in event_filter_match()
2262 WARN_ON_ONCE(event->ctx != ctx); in event_sched_out()
2263 lockdep_assert_held(&ctx->lock); in event_sched_out()
2265 if (event->state != PERF_EVENT_STATE_ACTIVE) in event_sched_out()
2273 list_del_init(&event->active_list); in event_sched_out()
2275 perf_pmu_disable(event->pmu); in event_sched_out()
2277 event->pmu->del(event, 0); in event_sched_out()
2278 event->oncpu = -1; in event_sched_out()
2280 if (event->pending_disable) { in event_sched_out()
2281 event->pending_disable = 0; in event_sched_out()
2286 if (event->pending_sigtrap) { in event_sched_out()
2289 event->pending_sigtrap = 0; in event_sched_out()
2291 !event->pending_work) { in event_sched_out()
2292 event->pending_work = 1; in event_sched_out()
2294 WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount)); in event_sched_out()
2295 task_work_add(current, &event->pending_task, TWA_RESUME); in event_sched_out()
2298 local_dec(&event->ctx->nr_pending); in event_sched_out()
2304 cpuctx->active_oncpu--; in event_sched_out()
2305 if (!--ctx->nr_active) in event_sched_out()
2307 if (event->attr.freq && event->attr.sample_freq) in event_sched_out()
2308 ctx->nr_freq--; in event_sched_out()
2309 if (event->attr.exclusive || !cpuctx->active_oncpu) in event_sched_out()
2310 cpuctx->exclusive = 0; in event_sched_out()
2312 perf_pmu_enable(event->pmu); in event_sched_out()
2322 if (group_event->state != PERF_EVENT_STATE_ACTIVE) in group_sched_out()
2325 perf_pmu_disable(ctx->pmu); in group_sched_out()
2335 perf_pmu_enable(ctx->pmu); in group_sched_out()
2356 if (ctx->is_active & EVENT_TIME) { in __perf_remove_from_context()
2366 event->pending_disable = 1; in __perf_remove_from_context()
2374 event->state = PERF_EVENT_STATE_DEAD; in __perf_remove_from_context()
2376 if (!ctx->nr_events && ctx->is_active) { in __perf_remove_from_context()
2377 if (ctx == &cpuctx->ctx) in __perf_remove_from_context()
2380 ctx->is_active = 0; in __perf_remove_from_context()
2381 ctx->rotate_necessary = 0; in __perf_remove_from_context()
2382 if (ctx->task) { in __perf_remove_from_context()
2383 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in __perf_remove_from_context()
2384 cpuctx->task_ctx = NULL; in __perf_remove_from_context()
2392 * If event->ctx is a cloned context, callers must make sure that
2393 * every task struct that event->ctx->task could possibly point to
2395 * that only calls us on the top-level context, which can't be a clone.
2401 struct perf_event_context *ctx = event->ctx; in perf_remove_from_context()
2403 lockdep_assert_held(&ctx->mutex); in perf_remove_from_context()
2410 raw_spin_lock_irq(&ctx->lock); in perf_remove_from_context()
2412 * Cgroup events are per-cpu events, and must IPI because of in perf_remove_from_context()
2415 if (!ctx->is_active && !is_cgroup_event(event)) { in perf_remove_from_context()
2418 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2421 raw_spin_unlock_irq(&ctx->lock); in perf_remove_from_context()
2434 if (event->state < PERF_EVENT_STATE_INACTIVE) in __perf_event_disable()
2437 if (ctx->is_active & EVENT_TIME) { in __perf_event_disable()
2442 if (event == event->group_leader) in __perf_event_disable()
2454 * If event->ctx is a cloned context, callers must make sure that
2455 * every task struct that event->ctx->task could possibly point to
2458 * hold the top-level event's child_mutex, so any descendant that
2461 * When called from perf_pending_irq it's OK because event->ctx
2467 struct perf_event_context *ctx = event->ctx; in _perf_event_disable()
2469 raw_spin_lock_irq(&ctx->lock); in _perf_event_disable()
2470 if (event->state <= PERF_EVENT_STATE_OFF) { in _perf_event_disable()
2471 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2474 raw_spin_unlock_irq(&ctx->lock); in _perf_event_disable()
2500 event->pending_disable = 1; in perf_event_disable_inatomic()
2501 irq_work_queue(&event->pending_irq); in perf_event_disable_inatomic()
2516 WARN_ON_ONCE(event->ctx != ctx); in event_sched_in()
2518 lockdep_assert_held(&ctx->lock); in event_sched_in()
2520 if (event->state <= PERF_EVENT_STATE_OFF) in event_sched_in()
2523 WRITE_ONCE(event->oncpu, smp_processor_id()); in event_sched_in()
2527 * ->oncpu if it sees ACTIVE. in event_sched_in()
2537 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { in event_sched_in()
2539 event->hw.interrupts = 0; in event_sched_in()
2542 perf_pmu_disable(event->pmu); in event_sched_in()
2546 if (event->pmu->add(event, PERF_EF_START)) { in event_sched_in()
2548 event->oncpu = -1; in event_sched_in()
2549 ret = -EAGAIN; in event_sched_in()
2554 cpuctx->active_oncpu++; in event_sched_in()
2555 if (!ctx->nr_active++) in event_sched_in()
2557 if (event->attr.freq && event->attr.sample_freq) in event_sched_in()
2558 ctx->nr_freq++; in event_sched_in()
2560 if (event->attr.exclusive) in event_sched_in()
2561 cpuctx->exclusive = 1; in event_sched_in()
2564 perf_pmu_enable(event->pmu); in event_sched_in()
2575 struct pmu *pmu = ctx->pmu; in group_sched_in()
2577 if (group_event->state == PERF_EVENT_STATE_OFF) in group_sched_in()
2580 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); in group_sched_in()
2595 if (!pmu->commit_txn(pmu)) in group_sched_in()
2613 pmu->cancel_txn(pmu); in group_sched_in()
2614 return -EAGAIN; in group_sched_in()
2627 if (event->group_caps & PERF_EV_CAP_SOFTWARE) in group_can_go_on()
2633 if (cpuctx->exclusive) in group_can_go_on()
2639 if (event->attr.exclusive && !list_empty(get_event_list(event))) in group_can_go_on()
2667 if (!cpuctx->task_ctx) in task_ctx_sched_out()
2670 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) in task_ctx_sched_out()
2689 * - CPU pinned (EVENT_CPU | EVENT_PINNED)
2690 * - task pinned (EVENT_PINNED)
2691 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE)
2692 * - task flexible (EVENT_FLEXIBLE).
2718 perf_pmu_disable(cpuctx->ctx.pmu); in ctx_resched()
2725 * - EVENT_CPU: schedule out corresponding groups; in ctx_resched()
2726 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; in ctx_resched()
2727 * - otherwise, do nothing more. in ctx_resched()
2735 perf_pmu_enable(cpuctx->ctx.pmu); in ctx_resched()
2740 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); in perf_pmu_resched()
2741 struct perf_event_context *task_ctx = cpuctx->task_ctx; in perf_pmu_resched()
2752 * things like ctx->is_active and cpuctx->task_ctx are set.
2757 struct perf_event_context *ctx = event->ctx; in __perf_install_in_context()
2759 struct perf_event_context *task_ctx = cpuctx->task_ctx; in __perf_install_in_context()
2763 raw_spin_lock(&cpuctx->ctx.lock); in __perf_install_in_context()
2764 if (ctx->task) { in __perf_install_in_context()
2765 raw_spin_lock(&ctx->lock); in __perf_install_in_context()
2768 reprogram = (ctx->task == current); in __perf_install_in_context()
2774 * If its not running, we don't care, ctx->lock will in __perf_install_in_context()
2777 if (task_curr(ctx->task) && !reprogram) { in __perf_install_in_context()
2778 ret = -ESRCH; in __perf_install_in_context()
2782 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); in __perf_install_in_context()
2784 raw_spin_lock(&task_ctx->lock); in __perf_install_in_context()
2788 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) { in __perf_install_in_context()
2794 reprogram = cgroup_is_descendant(cgrp->css.cgroup, in __perf_install_in_context()
2795 event->cgrp->css.cgroup); in __perf_install_in_context()
2826 struct task_struct *task = READ_ONCE(ctx->task); in perf_install_in_context()
2828 lockdep_assert_held(&ctx->mutex); in perf_install_in_context()
2832 if (event->cpu != -1) in perf_install_in_context()
2833 event->cpu = cpu; in perf_install_in_context()
2836 * Ensures that if we can observe event->ctx, both the event and ctx in perf_install_in_context()
2839 smp_store_release(&event->ctx, ctx); in perf_install_in_context()
2843 * without IPI. Except when this is the first event for the context, in in perf_install_in_context()
2844 * that case we need the magic of the IPI to set ctx->is_active. in perf_install_in_context()
2845 * Similarly, cgroup events for the context also needs the IPI to in perf_install_in_context()
2849 * event will issue the IPI and reprogram the hardware. in perf_install_in_context()
2852 ctx->nr_events && !is_cgroup_event(event)) { in perf_install_in_context()
2853 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
2854 if (ctx->task == TASK_TOMBSTONE) { in perf_install_in_context()
2855 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2859 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2875 * Installing events is tricky because we cannot rely on ctx->is_active in perf_install_in_context()
2876 * to be set in case this is the nr_events 0 -> 1 transition. in perf_install_in_context()
2890 * our task->perf_event_ctxp[] store, such that it will in fact take in perf_install_in_context()
2893 * We do this by task_function_call(), if the IPI fails to hit the task in perf_install_in_context()
2899 * This smp_mb() orders the task->perf_event_ctxp[] store with the in perf_install_in_context()
2900 * task_cpu() load, such that if the IPI then does not find the task in perf_install_in_context()
2909 raw_spin_lock_irq(&ctx->lock); in perf_install_in_context()
2910 task = ctx->task; in perf_install_in_context()
2914 * cannot happen), and we hold ctx->mutex, which serializes us in perf_install_in_context()
2917 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2921 * If the task is not running, ctx->lock will avoid it becoming so, in perf_install_in_context()
2925 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2929 raw_spin_unlock_irq(&ctx->lock); in perf_install_in_context()
2940 struct perf_event *leader = event->group_leader; in __perf_event_enable()
2943 if (event->state >= PERF_EVENT_STATE_INACTIVE || in __perf_event_enable()
2944 event->state <= PERF_EVENT_STATE_ERROR) in __perf_event_enable()
2947 if (ctx->is_active) in __perf_event_enable()
2953 if (!ctx->is_active) in __perf_event_enable()
2965 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { in __perf_event_enable()
2970 task_ctx = cpuctx->task_ctx; in __perf_event_enable()
2971 if (ctx->task) in __perf_event_enable()
2980 * If event->ctx is a cloned context, callers must make sure that
2981 * every task struct that event->ctx->task could possibly point to
2988 struct perf_event_context *ctx = event->ctx; in _perf_event_enable()
2990 raw_spin_lock_irq(&ctx->lock); in _perf_event_enable()
2991 if (event->state >= PERF_EVENT_STATE_INACTIVE || in _perf_event_enable()
2992 event->state < PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
2994 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3003 * been scheduled away before the cross-call arrived. in _perf_event_enable()
3005 if (event->state == PERF_EVENT_STATE_ERROR) { in _perf_event_enable()
3009 if (event->event_caps & PERF_EV_CAP_SIBLING && in _perf_event_enable()
3010 event->group_leader == event) in _perf_event_enable()
3013 event->state = PERF_EVENT_STATE_OFF; in _perf_event_enable()
3015 raw_spin_unlock_irq(&ctx->lock); in _perf_event_enable()
3041 struct perf_event *event = sd->event; in __perf_event_stop()
3044 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in __perf_event_stop()
3054 if (READ_ONCE(event->oncpu) != smp_processor_id()) in __perf_event_stop()
3055 return -EAGAIN; in __perf_event_stop()
3057 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_stop()
3065 * Since this is happening on an event-local CPU, no trace is lost in __perf_event_stop()
3068 if (sd->restart) in __perf_event_stop()
3069 event->pmu->start(event, 0); in __perf_event_stop()
3083 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) in perf_event_stop()
3091 * inactive here (event->oncpu==-1), there's nothing more to do; in perf_event_stop()
3092 * fall through with ret==-ENXIO. in perf_event_stop()
3094 ret = cpu_function_call(READ_ONCE(event->oncpu), in perf_event_stop()
3096 } while (ret == -EAGAIN); in perf_event_stop()
3115 * pre-existing mappings, called once when new filters arrive via SET_FILTER
3130 raw_spin_lock(&ifh->lock); in perf_event_addr_filters_sync()
3131 if (event->addr_filters_gen != event->hw.addr_filters_gen) { in perf_event_addr_filters_sync()
3132 event->pmu->addr_filters_sync(event); in perf_event_addr_filters_sync()
3133 event->hw.addr_filters_gen = event->addr_filters_gen; in perf_event_addr_filters_sync()
3135 raw_spin_unlock(&ifh->lock); in perf_event_addr_filters_sync()
3144 if (event->attr.inherit || !is_sampling_event(event)) in _perf_event_refresh()
3145 return -EINVAL; in _perf_event_refresh()
3147 atomic_add(refresh, &event->event_limit); in _perf_event_refresh()
3178 if (!bp->attr.disabled) in perf_event_modify_breakpoint()
3185 * Copy event-type-independent attributes that may be modified.
3190 to->sig_data = from->sig_data; in perf_event_modify_copy_attr()
3200 if (event->attr.type != attr->type) in perf_event_modify_attr()
3201 return -EINVAL; in perf_event_modify_attr()
3203 switch (event->attr.type) { in perf_event_modify_attr()
3209 return -EOPNOTSUPP; in perf_event_modify_attr()
3212 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_modify_attr()
3214 mutex_lock(&event->child_mutex); in perf_event_modify_attr()
3216 * Event-type-independent attributes must be copied before event-type in perf_event_modify_attr()
3220 perf_event_modify_copy_attr(&event->attr, attr); in perf_event_modify_attr()
3224 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_modify_attr()
3225 perf_event_modify_copy_attr(&child->attr, attr); in perf_event_modify_attr()
3231 mutex_unlock(&event->child_mutex); in perf_event_modify_attr()
3240 int is_active = ctx->is_active; in ctx_sched_out()
3242 lockdep_assert_held(&ctx->lock); in ctx_sched_out()
3244 if (likely(!ctx->nr_events)) { in ctx_sched_out()
3248 WARN_ON_ONCE(ctx->is_active); in ctx_sched_out()
3249 if (ctx->task) in ctx_sched_out()
3250 WARN_ON_ONCE(cpuctx->task_ctx); in ctx_sched_out()
3267 update_cgrp_time_from_cpuctx(cpuctx, ctx == &cpuctx->ctx); in ctx_sched_out()
3269 * CPU-release for the below ->is_active store, in ctx_sched_out()
3275 ctx->is_active &= ~event_type; in ctx_sched_out()
3276 if (!(ctx->is_active & EVENT_ALL)) in ctx_sched_out()
3277 ctx->is_active = 0; in ctx_sched_out()
3279 if (ctx->task) { in ctx_sched_out()
3280 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_out()
3281 if (!ctx->is_active) in ctx_sched_out()
3282 cpuctx->task_ctx = NULL; in ctx_sched_out()
3285 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_out()
3287 if (!ctx->nr_active || !(is_active & EVENT_ALL)) in ctx_sched_out()
3290 perf_pmu_disable(ctx->pmu); in ctx_sched_out()
3292 list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list) in ctx_sched_out()
3297 list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list) in ctx_sched_out()
3305 ctx->rotate_necessary = 0; in ctx_sched_out()
3307 perf_pmu_enable(ctx->pmu); in ctx_sched_out()
3321 lockdep_assert_held(&ctx1->lock); in context_equiv()
3322 lockdep_assert_held(&ctx2->lock); in context_equiv()
3325 if (ctx1->pin_count || ctx2->pin_count) in context_equiv()
3329 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) in context_equiv()
3333 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) in context_equiv()
3340 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && in context_equiv()
3341 ctx1->parent_gen == ctx2->parent_gen) in context_equiv()
3353 if (!event->attr.inherit_stat) in __perf_event_sync_stat()
3363 if (event->state == PERF_EVENT_STATE_ACTIVE) in __perf_event_sync_stat()
3364 event->pmu->read(event); in __perf_event_sync_stat()
3369 * In order to keep per-task stats reliable we need to flip the event in __perf_event_sync_stat()
3372 value = local64_read(&next_event->count); in __perf_event_sync_stat()
3373 value = local64_xchg(&event->count, value); in __perf_event_sync_stat()
3374 local64_set(&next_event->count, value); in __perf_event_sync_stat()
3376 swap(event->total_time_enabled, next_event->total_time_enabled); in __perf_event_sync_stat()
3377 swap(event->total_time_running, next_event->total_time_running); in __perf_event_sync_stat()
3391 if (!ctx->nr_stat) in perf_event_sync_stat()
3396 event = list_first_entry(&ctx->event_list, in perf_event_sync_stat()
3399 next_event = list_first_entry(&next_ctx->event_list, in perf_event_sync_stat()
3402 while (&event->event_entry != &ctx->event_list && in perf_event_sync_stat()
3403 &next_event->event_entry != &next_ctx->event_list) { in perf_event_sync_stat()
3415 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; in perf_event_context_sched_out()
3425 pmu = ctx->pmu; in perf_event_context_sched_out()
3427 if (!cpuctx->task_ctx) in perf_event_context_sched_out()
3431 next_ctx = next->perf_event_ctxp[ctxn]; in perf_event_context_sched_out()
3435 parent = rcu_dereference(ctx->parent_ctx); in perf_event_context_sched_out()
3436 next_parent = rcu_dereference(next_ctx->parent_ctx); in perf_event_context_sched_out()
3447 * lock (including re-checking that neither has been in perf_event_context_sched_out()
3452 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3453 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); in perf_event_context_sched_out()
3458 /* PMIs are disabled; ctx->nr_pending is stable. */ in perf_event_context_sched_out()
3459 if (local_read(&ctx->nr_pending) || in perf_event_context_sched_out()
3460 local_read(&next_ctx->nr_pending)) { in perf_event_context_sched_out()
3463 * events that rely on the ctx->task relation. in perf_event_context_sched_out()
3465 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3470 WRITE_ONCE(ctx->task, next); in perf_event_context_sched_out()
3471 WRITE_ONCE(next_ctx->task, task); in perf_event_context_sched_out()
3473 if (cpuctx->sched_cb_usage && pmu->sched_task) in perf_event_context_sched_out()
3474 pmu->sched_task(ctx, false); in perf_event_context_sched_out()
3482 if (pmu->swap_task_ctx) in perf_event_context_sched_out()
3483 pmu->swap_task_ctx(ctx, next_ctx); in perf_event_context_sched_out()
3485 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); in perf_event_context_sched_out()
3492 * ctx->task and ctx->task_ctx_data are immaterial in perf_event_context_sched_out()
3494 * ctx->lock which we're now holding. in perf_event_context_sched_out()
3496 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); in perf_event_context_sched_out()
3497 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); in perf_event_context_sched_out()
3503 raw_spin_unlock(&next_ctx->lock); in perf_event_context_sched_out()
3504 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3510 raw_spin_lock(&ctx->lock); in perf_event_context_sched_out()
3514 if (cpuctx->sched_cb_usage && pmu->sched_task) in perf_event_context_sched_out()
3515 pmu->sched_task(ctx, false); in perf_event_context_sched_out()
3519 raw_spin_unlock(&ctx->lock); in perf_event_context_sched_out()
3527 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); in perf_sched_cb_dec()
3531 if (!--cpuctx->sched_cb_usage) in perf_sched_cb_dec()
3532 list_del(&cpuctx->sched_cb_entry); in perf_sched_cb_dec()
3538 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); in perf_sched_cb_inc()
3540 if (!cpuctx->sched_cb_usage++) in perf_sched_cb_inc()
3541 list_add(&cpuctx->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); in perf_sched_cb_inc()
3550 * This callback is relevant even to per-cpu events; for example multi event
3558 pmu = cpuctx->ctx.pmu; /* software PMUs will not have sched_task */ in __perf_pmu_sched_task()
3560 if (WARN_ON_ONCE(!pmu->sched_task)) in __perf_pmu_sched_task()
3563 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3566 pmu->sched_task(cpuctx->task_ctx, sched_in); in __perf_pmu_sched_task()
3569 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in __perf_pmu_sched_task()
3583 if (cpuctx->task_ctx) in perf_pmu_sched_task()
3600 * We stop each event and update the event value in event->count.
3624 * cgroup event are system-wide mode only in __perf_event_task_sched_out()
3636 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); in cpu_ctx_sched_out()
3644 return le->group_index < re->group_index; in perf_less_group_idx()
3662 struct perf_event **itrs = heap->data; in __heap_add()
3665 itrs[heap->nr] = event; in __heap_add()
3666 heap->nr++; in __heap_add()
3686 .data = cpuctx->heap, in visit_groups_merge()
3688 .size = cpuctx->heap_size, in visit_groups_merge()
3691 lockdep_assert_held(&cpuctx->ctx.lock); in visit_groups_merge()
3694 if (cpuctx->cgrp) in visit_groups_merge()
3695 css = &cpuctx->cgrp->css; in visit_groups_merge()
3704 __heap_add(&event_heap, perf_event_groups_first(groups, -1, NULL)); in visit_groups_merge()
3711 for (; css; css = css->parent) in visit_groups_merge()
3712 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, css->cgroup)); in visit_groups_merge()
3733 * Because the userpage is strictly per-event (there is no concept of context,
3735 * when context time starts :-(
3741 if (likely(!atomic_read(&event->mmap_count))) in event_update_userpage()
3763 struct perf_event_context *ctx = event->ctx; in merge_sched_in()
3767 if (event->state <= PERF_EVENT_STATE_OFF) in merge_sched_in()
3775 list_add_tail(&event->active_list, get_event_list(event)); in merge_sched_in()
3778 if (event->state == PERF_EVENT_STATE_INACTIVE) { in merge_sched_in()
3780 if (event->attr.pinned) { in merge_sched_in()
3784 ctx->rotate_necessary = 1; in merge_sched_in()
3799 if (ctx != &cpuctx->ctx) in ctx_pinned_sched_in()
3802 visit_groups_merge(cpuctx, &ctx->pinned_groups, in ctx_pinned_sched_in()
3813 if (ctx != &cpuctx->ctx) in ctx_flexible_sched_in()
3816 visit_groups_merge(cpuctx, &ctx->flexible_groups, in ctx_flexible_sched_in()
3826 int is_active = ctx->is_active; in ctx_sched_in()
3828 lockdep_assert_held(&ctx->lock); in ctx_sched_in()
3830 if (likely(!ctx->nr_events)) in ctx_sched_in()
3838 * CPU-release for the below ->is_active store, in ctx_sched_in()
3844 ctx->is_active |= (event_type | EVENT_TIME); in ctx_sched_in()
3845 if (ctx->task) { in ctx_sched_in()
3847 cpuctx->task_ctx = ctx; in ctx_sched_in()
3849 WARN_ON_ONCE(cpuctx->task_ctx != ctx); in ctx_sched_in()
3852 is_active ^= ctx->is_active; /* changed bits */ in ctx_sched_in()
3869 struct perf_event_context *ctx = &cpuctx->ctx; in cpu_ctx_sched_in()
3886 pmu = ctx->pmu = cpuctx->ctx.pmu; in perf_event_context_sched_in()
3888 if (cpuctx->task_ctx == ctx) { in perf_event_context_sched_in()
3889 if (cpuctx->sched_cb_usage) in perf_event_context_sched_in()
3896 * We must check ctx->nr_events while holding ctx->lock, such in perf_event_context_sched_in()
3899 if (!ctx->nr_events) in perf_event_context_sched_in()
3911 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) in perf_event_context_sched_in()
3915 if (cpuctx->sched_cb_usage && pmu->sched_task) in perf_event_context_sched_in()
3916 pmu->sched_task(cpuctx->task_ctx, true); in perf_event_context_sched_in()
3942 ctx = task->perf_event_ctxp[ctxn]; in __perf_event_task_sched_in()
3958 u64 frequency = event->attr.sample_freq; in perf_calculate_period()
3974 * period = ------------------- in perf_calculate_period()
3987 a##_fls--; \ in perf_calculate_period()
3990 b##_fls--; \ in perf_calculate_period()
4034 struct hw_perf_event *hwc = &event->hw; in perf_adjust_period()
4040 delta = (s64)(period - hwc->sample_period); in perf_adjust_period()
4043 sample_period = hwc->sample_period + delta; in perf_adjust_period()
4048 hwc->sample_period = sample_period; in perf_adjust_period()
4050 if (local64_read(&hwc->period_left) > 8*sample_period) { in perf_adjust_period()
4052 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_period()
4054 local64_set(&hwc->period_left, 0); in perf_adjust_period()
4057 event->pmu->start(event, PERF_EF_RELOAD); in perf_adjust_period()
4076 * - context have events in frequency mode (needs freq adjust) in perf_adjust_freq_unthr_context()
4077 * - there are events to unthrottle on this cpu in perf_adjust_freq_unthr_context()
4079 if (!(ctx->nr_freq || needs_unthr)) in perf_adjust_freq_unthr_context()
4082 raw_spin_lock(&ctx->lock); in perf_adjust_freq_unthr_context()
4083 perf_pmu_disable(ctx->pmu); in perf_adjust_freq_unthr_context()
4085 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_adjust_freq_unthr_context()
4086 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_adjust_freq_unthr_context()
4092 perf_pmu_disable(event->pmu); in perf_adjust_freq_unthr_context()
4094 hwc = &event->hw; in perf_adjust_freq_unthr_context()
4096 if (hwc->interrupts == MAX_INTERRUPTS) { in perf_adjust_freq_unthr_context()
4097 hwc->interrupts = 0; in perf_adjust_freq_unthr_context()
4099 event->pmu->start(event, 0); in perf_adjust_freq_unthr_context()
4102 if (!event->attr.freq || !event->attr.sample_freq) in perf_adjust_freq_unthr_context()
4106 * stop the event and update event->count in perf_adjust_freq_unthr_context()
4108 event->pmu->stop(event, PERF_EF_UPDATE); in perf_adjust_freq_unthr_context()
4110 now = local64_read(&event->count); in perf_adjust_freq_unthr_context()
4111 delta = now - hwc->freq_count_stamp; in perf_adjust_freq_unthr_context()
4112 hwc->freq_count_stamp = now; in perf_adjust_freq_unthr_context()
4124 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); in perf_adjust_freq_unthr_context()
4126 perf_pmu_enable(event->pmu); in perf_adjust_freq_unthr_context()
4129 perf_pmu_enable(ctx->pmu); in perf_adjust_freq_unthr_context()
4130 raw_spin_unlock(&ctx->lock); in perf_adjust_freq_unthr_context()
4139 * Rotate the first entry last of non-pinned groups. Rotation might be in rotate_ctx()
4142 if (ctx->rotate_disable) in rotate_ctx()
4145 perf_event_groups_delete(&ctx->flexible_groups, event); in rotate_ctx()
4146 perf_event_groups_insert(&ctx->flexible_groups, event); in rotate_ctx()
4156 event = list_first_entry_or_null(&ctx->flexible_active, in ctx_event_to_rotate()
4161 event = rb_entry_safe(rb_first(&ctx->flexible_groups.tree), in ctx_event_to_rotate()
4169 ctx->rotate_necessary = 0; in ctx_event_to_rotate()
4185 cpu_rotate = cpuctx->ctx.rotate_necessary; in perf_rotate_context()
4186 task_ctx = cpuctx->task_ctx; in perf_rotate_context()
4187 task_rotate = task_ctx ? task_ctx->rotate_necessary : 0; in perf_rotate_context()
4192 perf_ctx_lock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4193 perf_pmu_disable(cpuctx->ctx.pmu); in perf_rotate_context()
4198 cpu_event = ctx_event_to_rotate(&cpuctx->ctx); in perf_rotate_context()
4212 rotate_ctx(&cpuctx->ctx, cpu_event); in perf_rotate_context()
4216 perf_pmu_enable(cpuctx->ctx.pmu); in perf_rotate_context()
4217 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); in perf_rotate_context()
4241 if (!event->attr.enable_on_exec) in event_enable_on_exec()
4244 event->attr.enable_on_exec = 0; in event_enable_on_exec()
4245 if (event->state >= PERF_EVENT_STATE_INACTIVE) in event_enable_on_exec()
4254 * Enable all of a task's events that have been marked enable-on-exec.
4267 ctx = current->perf_event_ctxp[ctxn]; in perf_event_enable_on_exec()
4268 if (!ctx || !ctx->nr_events) in perf_event_enable_on_exec()
4274 list_for_each_entry(event, &ctx->event_list, event_entry) { in perf_event_enable_on_exec()
4303 * remove-on-exec, and feeds their values back to parent events.
4316 mutex_lock(&ctx->mutex); in perf_event_remove_on_exec()
4318 if (WARN_ON_ONCE(ctx->task != current)) in perf_event_remove_on_exec()
4321 list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { in perf_event_remove_on_exec()
4322 if (!event->attr.remove_on_exec) in perf_event_remove_on_exec()
4333 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_remove_on_exec()
4336 --ctx->pin_count; in perf_event_remove_on_exec()
4337 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_remove_on_exec()
4340 mutex_unlock(&ctx->mutex); in perf_event_remove_on_exec()
4357 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { in __perf_event_read_cpu()
4376 struct perf_event *sub, *event = data->event; in __perf_event_read()
4377 struct perf_event_context *ctx = event->ctx; in __perf_event_read()
4379 struct pmu *pmu = event->pmu; in __perf_event_read()
4385 * event->count would have been updated to a recent sample in __perf_event_read()
4388 if (ctx->task && cpuctx->task_ctx != ctx) in __perf_event_read()
4391 raw_spin_lock(&ctx->lock); in __perf_event_read()
4392 if (ctx->is_active & EVENT_TIME) { in __perf_event_read()
4398 if (data->group) in __perf_event_read()
4401 if (event->state != PERF_EVENT_STATE_ACTIVE) in __perf_event_read()
4404 if (!data->group) { in __perf_event_read()
4405 pmu->read(event); in __perf_event_read()
4406 data->ret = 0; in __perf_event_read()
4410 pmu->start_txn(pmu, PERF_PMU_TXN_READ); in __perf_event_read()
4412 pmu->read(event); in __perf_event_read()
4415 if (sub->state == PERF_EVENT_STATE_ACTIVE) { in __perf_event_read()
4420 sub->pmu->read(sub); in __perf_event_read()
4424 data->ret = pmu->commit_txn(pmu); in __perf_event_read()
4427 raw_spin_unlock(&ctx->lock); in __perf_event_read()
4432 return local64_read(&event->count) + atomic64_read(&event->child_count); in perf_event_count()
4448 * NMI-safe method to read a local event, that is an event that
4450 * - either for the current task, or for this CPU
4451 * - does not have inherit set, for inherited task events
4453 * - must not have a pmu::count method
4471 if (event->attr.inherit) { in perf_event_read_local()
4472 ret = -EOPNOTSUPP; in perf_event_read_local()
4476 /* If this is a per-task event, it must be for current */ in perf_event_read_local()
4477 if ((event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4478 event->hw.target != current) { in perf_event_read_local()
4479 ret = -EINVAL; in perf_event_read_local()
4483 /* If this is a per-CPU event, it must be for this CPU */ in perf_event_read_local()
4484 if (!(event->attach_state & PERF_ATTACH_TASK) && in perf_event_read_local()
4485 event->cpu != smp_processor_id()) { in perf_event_read_local()
4486 ret = -EINVAL; in perf_event_read_local()
4491 if (event->attr.pinned && event->oncpu != smp_processor_id()) { in perf_event_read_local()
4492 ret = -EBUSY; in perf_event_read_local()
4497 * If the event is currently on this CPU, its either a per-task event, in perf_event_read_local()
4499 * oncpu == -1). in perf_event_read_local()
4501 if (event->oncpu == smp_processor_id()) in perf_event_read_local()
4502 event->pmu->read(event); in perf_event_read_local()
4504 *value = local64_read(&event->count); in perf_event_read_local()
4522 enum perf_event_state state = READ_ONCE(event->state); in perf_event_read()
4534 * Orders the ->state and ->oncpu loads such that if we see in perf_event_read()
4535 * ACTIVE we must also see the right ->oncpu. in perf_event_read()
4541 event_cpu = READ_ONCE(event->oncpu); in perf_event_read()
4561 * Therefore, either way, we'll have an up-to-date event count in perf_event_read()
4569 struct perf_event_context *ctx = event->ctx; in perf_event_read()
4572 raw_spin_lock_irqsave(&ctx->lock, flags); in perf_event_read()
4573 state = event->state; in perf_event_read()
4575 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4583 if (ctx->is_active & EVENT_TIME) { in perf_event_read()
4591 raw_spin_unlock_irqrestore(&ctx->lock, flags); in perf_event_read()
4602 raw_spin_lock_init(&ctx->lock); in __perf_event_init_context()
4603 mutex_init(&ctx->mutex); in __perf_event_init_context()
4604 INIT_LIST_HEAD(&ctx->active_ctx_list); in __perf_event_init_context()
4605 perf_event_groups_init(&ctx->pinned_groups); in __perf_event_init_context()
4606 perf_event_groups_init(&ctx->flexible_groups); in __perf_event_init_context()
4607 INIT_LIST_HEAD(&ctx->event_list); in __perf_event_init_context()
4608 INIT_LIST_HEAD(&ctx->pinned_active); in __perf_event_init_context()
4609 INIT_LIST_HEAD(&ctx->flexible_active); in __perf_event_init_context()
4610 refcount_set(&ctx->refcount, 1); in __perf_event_init_context()
4624 ctx->task = get_task_struct(task); in alloc_perf_context()
4625 ctx->pmu = pmu; in alloc_perf_context()
4645 return ERR_PTR(-ESRCH); in find_lively_task_by_vpid()
4662 int cpu = event->cpu; in find_get_context()
4666 err = perf_allow_cpu(&event->attr); in find_get_context()
4670 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); in find_get_context()
4671 ctx = &cpuctx->ctx; in find_get_context()
4673 raw_spin_lock_irqsave(&ctx->lock, flags); in find_get_context()
4674 ++ctx->pin_count; in find_get_context()
4675 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4680 err = -EINVAL; in find_get_context()
4681 ctxn = pmu->task_ctx_nr; in find_get_context()
4685 if (event->attach_state & PERF_ATTACH_TASK_DATA) { in find_get_context()
4688 err = -ENOMEM; in find_get_context()
4697 ++ctx->pin_count; in find_get_context()
4699 if (task_ctx_data && !ctx->task_ctx_data) { in find_get_context()
4700 ctx->task_ctx_data = task_ctx_data; in find_get_context()
4703 raw_spin_unlock_irqrestore(&ctx->lock, flags); in find_get_context()
4709 err = -ENOMEM; in find_get_context()
4714 ctx->task_ctx_data = task_ctx_data; in find_get_context()
4719 mutex_lock(&task->perf_event_mutex); in find_get_context()
4724 if (task->flags & PF_EXITING) in find_get_context()
4725 err = -ESRCH; in find_get_context()
4726 else if (task->perf_event_ctxp[ctxn]) in find_get_context()
4727 err = -EAGAIN; in find_get_context()
4730 ++ctx->pin_count; in find_get_context()
4731 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); in find_get_context()
4733 mutex_unlock(&task->perf_event_mutex); in find_get_context()
4738 if (err == -EAGAIN) in find_get_context()
4759 if (event->ns) in free_event_rcu()
4760 put_pid_ns(event->ns); in free_event_rcu()
4770 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in detach_sb_event()
4772 raw_spin_lock(&pel->lock); in detach_sb_event()
4773 list_del_rcu(&event->sb_list); in detach_sb_event()
4774 raw_spin_unlock(&pel->lock); in detach_sb_event()
4779 struct perf_event_attr *attr = &event->attr; in is_sb_event()
4781 if (event->parent) in is_sb_event()
4784 if (event->attach_state & PERF_ATTACH_TASK) in is_sb_event()
4787 if (attr->mmap || attr->mmap_data || attr->mmap2 || in is_sb_event()
4788 attr->comm || attr->comm_exec || in is_sb_event()
4789 attr->task || attr->ksymbol || in is_sb_event()
4790 attr->context_switch || attr->text_poke || in is_sb_event()
4791 attr->bpf_event) in is_sb_event()
4804 if (event->parent) in unaccount_event_cpu()
4837 if (event->parent) in unaccount_event()
4840 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in unaccount_event()
4842 if (event->attr.mmap || event->attr.mmap_data) in unaccount_event()
4844 if (event->attr.build_id) in unaccount_event()
4846 if (event->attr.comm) in unaccount_event()
4848 if (event->attr.namespaces) in unaccount_event()
4850 if (event->attr.cgroup) in unaccount_event()
4852 if (event->attr.task) in unaccount_event()
4854 if (event->attr.freq) in unaccount_event()
4856 if (event->attr.context_switch) { in unaccount_event()
4864 if (event->attr.ksymbol) in unaccount_event()
4866 if (event->attr.bpf_event) in unaccount_event()
4868 if (event->attr.text_poke) in unaccount_event()
4872 if (!atomic_add_unless(&perf_sched_count, -1, 1)) in unaccount_event()
4876 unaccount_event_cpu(event, event->cpu); in unaccount_event()
4894 * 1) cpu-wide events in the presence of per-task events,
4895 * 2) per-task events in the presence of cpu-wide events,
4899 * _free_event()), the latter -- before the first perf_install_in_context().
4903 struct pmu *pmu = event->pmu; in exclusive_event_init()
4909 * Prevent co-existence of per-task and cpu-wide events on the in exclusive_event_init()
4912 * Negative pmu::exclusive_cnt means there are cpu-wide in exclusive_event_init()
4914 * per-task events. in exclusive_event_init()
4918 * to mean "per-task event", because unlike other attach states it in exclusive_event_init()
4921 if (event->attach_state & PERF_ATTACH_TASK) { in exclusive_event_init()
4922 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) in exclusive_event_init()
4923 return -EBUSY; in exclusive_event_init()
4925 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) in exclusive_event_init()
4926 return -EBUSY; in exclusive_event_init()
4934 struct pmu *pmu = event->pmu; in exclusive_event_destroy()
4940 if (event->attach_state & PERF_ATTACH_TASK) in exclusive_event_destroy()
4941 atomic_dec(&pmu->exclusive_cnt); in exclusive_event_destroy()
4943 atomic_inc(&pmu->exclusive_cnt); in exclusive_event_destroy()
4948 if ((e1->pmu == e2->pmu) && in exclusive_event_match()
4949 (e1->cpu == e2->cpu || in exclusive_event_match()
4950 e1->cpu == -1 || in exclusive_event_match()
4951 e2->cpu == -1)) in exclusive_event_match()
4960 struct pmu *pmu = event->pmu; in exclusive_event_installable()
4962 lockdep_assert_held(&ctx->mutex); in exclusive_event_installable()
4967 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { in exclusive_event_installable()
4980 irq_work_sync(&event->pending_irq); in _free_event()
4986 if (event->rb) { in _free_event()
4988 * Can happen when we close an event with re-directed output. in _free_event()
4993 mutex_lock(&event->mmap_mutex); in _free_event()
4995 mutex_unlock(&event->mmap_mutex); in _free_event()
5001 if (!event->parent) { in _free_event()
5002 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) in _free_event()
5008 kfree(event->addr_filter_ranges); in _free_event()
5010 if (event->destroy) in _free_event()
5011 event->destroy(event); in _free_event()
5014 * Must be after ->destroy(), due to uprobe_perf_close() using in _free_event()
5017 if (event->hw.target) in _free_event()
5018 put_task_struct(event->hw.target); in _free_event()
5024 if (event->ctx) in _free_event()
5025 put_ctx(event->ctx); in _free_event()
5028 module_put(event->pmu->module); in _free_event()
5030 call_rcu(&event->rcu_head, free_event_rcu); in _free_event()
5039 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, in free_event()
5041 atomic_long_read(&event->refcount), event)) { in free_event()
5042 /* leak to avoid use-after-free */ in free_event()
5061 * owner->perf_event_mutex. in perf_remove_from_owner()
5063 owner = READ_ONCE(event->owner); in perf_remove_from_owner()
5077 * holding ctx->mutex which would be an inversion wrt. the in perf_remove_from_owner()
5081 * ctx->mutex. in perf_remove_from_owner()
5083 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); in perf_remove_from_owner()
5086 * We have to re-check the event->owner field, if it is cleared in perf_remove_from_owner()
5091 if (event->owner) { in perf_remove_from_owner()
5092 list_del_init(&event->owner_entry); in perf_remove_from_owner()
5093 smp_store_release(&event->owner, NULL); in perf_remove_from_owner()
5095 mutex_unlock(&owner->perf_event_mutex); in perf_remove_from_owner()
5102 if (!atomic_long_dec_and_test(&event->refcount)) in put_event()
5115 struct perf_event_context *ctx = event->ctx; in perf_event_release_kernel()
5124 WARN_ON_ONCE(event->attach_state & in perf_event_release_kernel()
5133 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_release_kernel()
5139 * Anybody acquiring event->child_mutex after the below loop _must_ in perf_event_release_kernel()
5151 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5152 list_for_each_entry(child, &event->child_list, child_list) { in perf_event_release_kernel()
5158 ctx = READ_ONCE(child->ctx); in perf_event_release_kernel()
5172 * can re-acquire child_mutex. in perf_event_release_kernel()
5174 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5175 mutex_lock(&ctx->mutex); in perf_event_release_kernel()
5176 mutex_lock(&event->child_mutex); in perf_event_release_kernel()
5183 tmp = list_first_entry_or_null(&event->child_list, in perf_event_release_kernel()
5187 list_move(&child->child_list, &free_list); in perf_event_release_kernel()
5195 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5196 mutex_unlock(&ctx->mutex); in perf_event_release_kernel()
5200 mutex_unlock(&event->child_mutex); in perf_event_release_kernel()
5203 void *var = &child->ctx->refcount; in perf_event_release_kernel()
5205 list_del(&child->child_list); in perf_event_release_kernel()
5227 perf_event_release_kernel(file->private_data); in perf_release()
5239 mutex_lock(&event->child_mutex); in __perf_event_read_value()
5244 *enabled += event->total_time_enabled + in __perf_event_read_value()
5245 atomic64_read(&event->child_total_time_enabled); in __perf_event_read_value()
5246 *running += event->total_time_running + in __perf_event_read_value()
5247 atomic64_read(&event->child_total_time_running); in __perf_event_read_value()
5249 list_for_each_entry(child, &event->child_list, child_list) { in __perf_event_read_value()
5252 *enabled += child->total_time_enabled; in __perf_event_read_value()
5253 *running += child->total_time_running; in __perf_event_read_value()
5255 mutex_unlock(&event->child_mutex); in __perf_event_read_value()
5276 struct perf_event_context *ctx = leader->ctx; in __perf_read_group_add()
5286 raw_spin_lock_irqsave(&ctx->lock, flags); in __perf_read_group_add()
5289 * Since we co-schedule groups, {enabled,running} times of siblings in __perf_read_group_add()
5294 values[n++] += leader->total_time_enabled + in __perf_read_group_add()
5295 atomic64_read(&leader->child_total_time_enabled); in __perf_read_group_add()
5299 values[n++] += leader->total_time_running + in __perf_read_group_add()
5300 atomic64_read(&leader->child_total_time_running); in __perf_read_group_add()
5310 values[n++] = atomic64_read(&leader->lost_samples); in __perf_read_group_add()
5317 values[n++] = atomic64_read(&sub->lost_samples); in __perf_read_group_add()
5320 raw_spin_unlock_irqrestore(&ctx->lock, flags); in __perf_read_group_add()
5327 struct perf_event *leader = event->group_leader, *child; in perf_read_group()
5328 struct perf_event_context *ctx = leader->ctx; in perf_read_group()
5332 lockdep_assert_held(&ctx->mutex); in perf_read_group()
5334 values = kzalloc(event->read_size, GFP_KERNEL); in perf_read_group()
5336 return -ENOMEM; in perf_read_group()
5338 values[0] = 1 + leader->nr_siblings; in perf_read_group()
5344 mutex_lock(&leader->child_mutex); in perf_read_group()
5350 list_for_each_entry(child, &leader->child_list, child_list) { in perf_read_group()
5356 mutex_unlock(&leader->child_mutex); in perf_read_group()
5358 ret = event->read_size; in perf_read_group()
5359 if (copy_to_user(buf, values, event->read_size)) in perf_read_group()
5360 ret = -EFAULT; in perf_read_group()
5364 mutex_unlock(&leader->child_mutex); in perf_read_group()
5385 values[n++] = atomic64_read(&event->lost_samples); in perf_read_one()
5388 return -EFAULT; in perf_read_one()
5397 if (event->state > PERF_EVENT_STATE_EXIT) in is_event_hup()
5400 mutex_lock(&event->child_mutex); in is_event_hup()
5401 no_children = list_empty(&event->child_list); in is_event_hup()
5402 mutex_unlock(&event->child_mutex); in is_event_hup()
5407 * Read the performance event - simple non blocking version for now
5412 u64 read_format = event->attr.read_format; in __perf_read()
5416 * Return end-of-file for a read on an event that is in in __perf_read()
5420 if (event->state == PERF_EVENT_STATE_ERROR) in __perf_read()
5423 if (count < event->read_size) in __perf_read()
5424 return -ENOSPC; in __perf_read()
5426 WARN_ON_ONCE(event->ctx->parent_ctx); in __perf_read()
5438 struct perf_event *event = file->private_data; in perf_read()
5455 struct perf_event *event = file->private_data; in perf_poll()
5459 poll_wait(file, &event->waitq, wait); in perf_poll()
5465 * Pin the event->rb by taking event->mmap_mutex; otherwise in perf_poll()
5468 mutex_lock(&event->mmap_mutex); in perf_poll()
5469 rb = event->rb; in perf_poll()
5471 events = atomic_xchg(&rb->poll, 0); in perf_poll()
5472 mutex_unlock(&event->mmap_mutex); in perf_poll()
5479 local64_set(&event->count, 0); in _perf_event_reset()
5490 WARN_ON_ONCE(event->attr.inherit); in perf_event_pause()
5492 count = local64_read(&event->count); in perf_event_pause()
5494 local64_set(&event->count, 0); in perf_event_pause()
5502 * Holding the top-level event's child_mutex means that any
5512 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_event_for_each_child()
5514 mutex_lock(&event->child_mutex); in perf_event_for_each_child()
5516 list_for_each_entry(child, &event->child_list, child_list) in perf_event_for_each_child()
5518 mutex_unlock(&event->child_mutex); in perf_event_for_each_child()
5524 struct perf_event_context *ctx = event->ctx; in perf_event_for_each()
5527 lockdep_assert_held(&ctx->mutex); in perf_event_for_each()
5529 event = event->group_leader; in perf_event_for_each()
5544 if (event->attr.freq) { in __perf_event_period()
5545 event->attr.sample_freq = value; in __perf_event_period()
5547 event->attr.sample_period = value; in __perf_event_period()
5548 event->hw.sample_period = value; in __perf_event_period()
5551 active = (event->state == PERF_EVENT_STATE_ACTIVE); in __perf_event_period()
5553 perf_pmu_disable(ctx->pmu); in __perf_event_period()
5556 * trying to unthrottle while we already re-started the event. in __perf_event_period()
5558 if (event->hw.interrupts == MAX_INTERRUPTS) { in __perf_event_period()
5559 event->hw.interrupts = 0; in __perf_event_period()
5562 event->pmu->stop(event, PERF_EF_UPDATE); in __perf_event_period()
5565 local64_set(&event->hw.period_left, 0); in __perf_event_period()
5568 event->pmu->start(event, PERF_EF_RELOAD); in __perf_event_period()
5569 perf_pmu_enable(ctx->pmu); in __perf_event_period()
5575 return event->pmu->check_period(event, value); in perf_event_check_period()
5581 return -EINVAL; in _perf_event_period()
5584 return -EINVAL; in _perf_event_period()
5586 if (event->attr.freq && value > sysctl_perf_event_sample_rate) in _perf_event_period()
5587 return -EINVAL; in _perf_event_period()
5590 return -EINVAL; in _perf_event_period()
5592 if (!event->attr.freq && (value & (1ULL << 63))) in _perf_event_period()
5593 return -EINVAL; in _perf_event_period()
5619 return -EBADF; in perf_fget_light()
5621 if (f.file->f_op != &perf_fops) { in perf_fget_light()
5623 return -EBADF; in perf_fget_light()
5659 return -EFAULT; in _perf_ioctl()
5668 return -EFAULT; in _perf_ioctl()
5675 if (arg != -1) { in _perf_ioctl()
5681 output_event = output.file->private_data; in _perf_ioctl()
5715 rb = rcu_dereference(event->rb); in _perf_ioctl()
5716 if (!rb || !rb->nr_pages) { in _perf_ioctl()
5718 return -EINVAL; in _perf_ioctl()
5739 return -ENOTTY; in _perf_ioctl()
5752 struct perf_event *event = file->private_data; in perf_ioctl()
5777 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ in perf_compat_ioctl()
5795 mutex_lock(¤t->perf_event_mutex); in perf_event_task_enable()
5796 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_enable()
5801 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_enable()
5811 mutex_lock(¤t->perf_event_mutex); in perf_event_task_disable()
5812 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { in perf_event_task_disable()
5817 mutex_unlock(¤t->perf_event_mutex); in perf_event_task_disable()
5824 if (event->hw.state & PERF_HES_STOPPED) in perf_event_index()
5827 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_event_index()
5830 return event->pmu->event_idx(event); in perf_event_index()
5839 rb = rcu_dereference(event->rb); in perf_event_init_userpage()
5843 userpg = rb->user_page; in perf_event_init_userpage()
5846 userpg->cap_bit0_is_deprecated = 1; in perf_event_init_userpage()
5847 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); in perf_event_init_userpage()
5848 userpg->data_offset = PAGE_SIZE; in perf_event_init_userpage()
5849 userpg->data_size = perf_data_size(rb); in perf_event_init_userpage()
5872 rb = rcu_dereference(event->rb); in perf_event_update_userpage()
5887 userpg = rb->user_page; in perf_event_update_userpage()
5893 ++userpg->lock; in perf_event_update_userpage()
5895 userpg->index = perf_event_index(event); in perf_event_update_userpage()
5896 userpg->offset = perf_event_count(event); in perf_event_update_userpage()
5897 if (userpg->index) in perf_event_update_userpage()
5898 userpg->offset -= local64_read(&event->hw.prev_count); in perf_event_update_userpage()
5900 userpg->time_enabled = enabled + in perf_event_update_userpage()
5901 atomic64_read(&event->child_total_time_enabled); in perf_event_update_userpage()
5903 userpg->time_running = running + in perf_event_update_userpage()
5904 atomic64_read(&event->child_total_time_running); in perf_event_update_userpage()
5909 ++userpg->lock; in perf_event_update_userpage()
5918 struct perf_event *event = vmf->vma->vm_file->private_data; in perf_mmap_fault()
5922 if (vmf->flags & FAULT_FLAG_MKWRITE) { in perf_mmap_fault()
5923 if (vmf->pgoff == 0) in perf_mmap_fault()
5929 rb = rcu_dereference(event->rb); in perf_mmap_fault()
5933 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) in perf_mmap_fault()
5936 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); in perf_mmap_fault()
5937 if (!vmf->page) in perf_mmap_fault()
5940 get_page(vmf->page); in perf_mmap_fault()
5941 vmf->page->mapping = vmf->vma->vm_file->f_mapping; in perf_mmap_fault()
5942 vmf->page->index = vmf->pgoff; in perf_mmap_fault()
5957 WARN_ON_ONCE(event->parent); in ring_buffer_attach()
5959 if (event->rb) { in ring_buffer_attach()
5962 * event->rb_entry and wait/clear when adding event->rb_entry. in ring_buffer_attach()
5964 WARN_ON_ONCE(event->rcu_pending); in ring_buffer_attach()
5966 old_rb = event->rb; in ring_buffer_attach()
5967 spin_lock_irqsave(&old_rb->event_lock, flags); in ring_buffer_attach()
5968 list_del_rcu(&event->rb_entry); in ring_buffer_attach()
5969 spin_unlock_irqrestore(&old_rb->event_lock, flags); in ring_buffer_attach()
5971 event->rcu_batches = get_state_synchronize_rcu(); in ring_buffer_attach()
5972 event->rcu_pending = 1; in ring_buffer_attach()
5976 if (event->rcu_pending) { in ring_buffer_attach()
5977 cond_synchronize_rcu(event->rcu_batches); in ring_buffer_attach()
5978 event->rcu_pending = 0; in ring_buffer_attach()
5981 spin_lock_irqsave(&rb->event_lock, flags); in ring_buffer_attach()
5982 list_add_rcu(&event->rb_entry, &rb->event_list); in ring_buffer_attach()
5983 spin_unlock_irqrestore(&rb->event_lock, flags); in ring_buffer_attach()
5993 * mid-air, but then again, whoever does it like this is in ring_buffer_attach()
5999 rcu_assign_pointer(event->rb, rb); in ring_buffer_attach()
6008 wake_up_all(&event->waitq); in ring_buffer_attach()
6016 if (event->parent) in ring_buffer_wakeup()
6017 event = event->parent; in ring_buffer_wakeup()
6020 rb = rcu_dereference(event->rb); in ring_buffer_wakeup()
6022 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) in ring_buffer_wakeup()
6023 wake_up_all(&event->waitq); in ring_buffer_wakeup()
6032 if (event->parent) in ring_buffer_get()
6033 event = event->parent; in ring_buffer_get()
6036 rb = rcu_dereference(event->rb); in ring_buffer_get()
6038 if (!refcount_inc_not_zero(&rb->refcount)) in ring_buffer_get()
6048 if (!refcount_dec_and_test(&rb->refcount)) in ring_buffer_put()
6051 WARN_ON_ONCE(!list_empty(&rb->event_list)); in ring_buffer_put()
6053 call_rcu(&rb->rcu_head, rb_free_rcu); in ring_buffer_put()
6058 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_open()
6060 atomic_inc(&event->mmap_count); in perf_mmap_open()
6061 atomic_inc(&event->rb->mmap_count); in perf_mmap_open()
6063 if (vma->vm_pgoff) in perf_mmap_open()
6064 atomic_inc(&event->rb->aux_mmap_count); in perf_mmap_open()
6066 if (event->pmu->event_mapped) in perf_mmap_open()
6067 event->pmu->event_mapped(event, vma->vm_mm); in perf_mmap_open()
6082 struct perf_event *event = vma->vm_file->private_data; in perf_mmap_close()
6084 struct user_struct *mmap_user = rb->mmap_user; in perf_mmap_close()
6085 int mmap_locked = rb->mmap_locked; in perf_mmap_close()
6089 if (event->pmu->event_unmapped) in perf_mmap_close()
6090 event->pmu->event_unmapped(event, vma->vm_mm); in perf_mmap_close()
6093 * rb->aux_mmap_count will always drop before rb->mmap_count and in perf_mmap_close()
6094 * event->mmap_count, so it is ok to use event->mmap_mutex to in perf_mmap_close()
6097 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && in perf_mmap_close()
6098 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { in perf_mmap_close()
6108 atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->locked_vm); in perf_mmap_close()
6109 atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6113 WARN_ON_ONCE(refcount_read(&rb->aux_refcount)); in perf_mmap_close()
6115 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6118 if (atomic_dec_and_test(&rb->mmap_count)) in perf_mmap_close()
6121 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) in perf_mmap_close()
6125 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6138 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { in perf_mmap_close()
6139 if (!atomic_long_inc_not_zero(&event->refcount)) { in perf_mmap_close()
6141 * This event is en-route to free_event() which will in perf_mmap_close()
6148 mutex_lock(&event->mmap_mutex); in perf_mmap_close()
6159 if (event->rb == rb) in perf_mmap_close()
6162 mutex_unlock(&event->mmap_mutex); in perf_mmap_close()
6174 * It could be there's still a few 0-ref events on the list; they'll in perf_mmap_close()
6175 * get cleaned up by free_event() -- they'll also still have their in perf_mmap_close()
6182 atomic_long_sub((size >> PAGE_SHIFT) + 1 - mmap_locked, in perf_mmap_close()
6183 &mmap_user->locked_vm); in perf_mmap_close()
6184 atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm); in perf_mmap_close()
6200 struct perf_event *event = file->private_data; in perf_mmap()
6211 * Don't allow mmap() of inherited per-task counters. This would in perf_mmap()
6215 if (event->cpu == -1 && event->attr.inherit) in perf_mmap()
6216 return -EINVAL; in perf_mmap()
6218 if (!(vma->vm_flags & VM_SHARED)) in perf_mmap()
6219 return -EINVAL; in perf_mmap()
6225 vma_size = vma->vm_end - vma->vm_start; in perf_mmap()
6227 if (vma->vm_pgoff == 0) { in perf_mmap()
6228 nr_pages = (vma_size / PAGE_SIZE) - 1; in perf_mmap()
6231 * AUX area mapping: if rb->aux_nr_pages != 0, it's already in perf_mmap()
6237 if (!event->rb) in perf_mmap()
6238 return -EINVAL; in perf_mmap()
6242 mutex_lock(&event->mmap_mutex); in perf_mmap()
6243 ret = -EINVAL; in perf_mmap()
6245 rb = event->rb; in perf_mmap()
6249 aux_offset = READ_ONCE(rb->user_page->aux_offset); in perf_mmap()
6250 aux_size = READ_ONCE(rb->user_page->aux_size); in perf_mmap()
6255 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) in perf_mmap()
6259 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) in perf_mmap()
6266 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) in perf_mmap()
6272 if (!atomic_inc_not_zero(&rb->mmap_count)) in perf_mmap()
6276 atomic_inc(&rb->aux_mmap_count); in perf_mmap()
6281 atomic_set(&rb->aux_mmap_count, 1); in perf_mmap()
6288 * If we have rb pages ensure they're a power-of-two number, so we in perf_mmap()
6292 return -EINVAL; in perf_mmap()
6295 return -EINVAL; in perf_mmap()
6297 WARN_ON_ONCE(event->ctx->parent_ctx); in perf_mmap()
6299 mutex_lock(&event->mmap_mutex); in perf_mmap()
6300 if (event->rb) { in perf_mmap()
6301 if (data_page_nr(event->rb) != nr_pages) { in perf_mmap()
6302 ret = -EINVAL; in perf_mmap()
6306 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { in perf_mmap()
6312 mutex_unlock(&event->mmap_mutex); in perf_mmap()
6322 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); in perf_mmap()
6329 user_locked = atomic_long_read(&user->locked_vm); in perf_mmap()
6333 * user->locked_vm > user_lock_limit in perf_mmap()
6344 extra = user_locked - user_lock_limit; in perf_mmap()
6345 user_extra -= extra; in perf_mmap()
6350 locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra; in perf_mmap()
6354 ret = -EPERM; in perf_mmap()
6358 WARN_ON(!rb && event->rb); in perf_mmap()
6360 if (vma->vm_flags & VM_WRITE) in perf_mmap()
6365 event->attr.watermark ? event->attr.wakeup_watermark : 0, in perf_mmap()
6366 event->cpu, flags); in perf_mmap()
6369 ret = -ENOMEM; in perf_mmap()
6373 atomic_set(&rb->mmap_count, 1); in perf_mmap()
6374 rb->mmap_user = get_current_user(); in perf_mmap()
6375 rb->mmap_locked = extra; in perf_mmap()
6383 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, in perf_mmap()
6384 event->attr.aux_watermark, flags); in perf_mmap()
6386 rb->aux_mmap_locked = extra; in perf_mmap()
6391 atomic_long_add(user_extra, &user->locked_vm); in perf_mmap()
6392 atomic64_add(extra, &vma->vm_mm->pinned_vm); in perf_mmap()
6394 atomic_inc(&event->mmap_count); in perf_mmap()
6396 atomic_dec(&rb->mmap_count); in perf_mmap()
6399 mutex_unlock(&event->mmap_mutex); in perf_mmap()
6405 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; in perf_mmap()
6406 vma->vm_ops = &perf_mmap_vmops; in perf_mmap()
6408 if (event->pmu->event_mapped) in perf_mmap()
6409 event->pmu->event_mapped(event, vma->vm_mm); in perf_mmap()
6417 struct perf_event *event = filp->private_data; in perf_fasync()
6421 retval = fasync_helper(fd, filp, on, &event->fasync); in perf_fasync()
6445 * to user-space before waking everybody up.
6451 if (event->parent) in perf_event_fasync()
6452 event = event->parent; in perf_event_fasync()
6453 return &event->fasync; in perf_event_fasync()
6460 if (event->pending_kill) { in perf_event_wakeup()
6461 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); in perf_event_wakeup()
6462 event->pending_kill = 0; in perf_event_wakeup()
6470 * ctx->task or current has changed in the meantime. This can be the in perf_sigtrap()
6473 if (WARN_ON_ONCE(event->ctx->task != current)) in perf_sigtrap()
6480 if (current->flags & PF_EXITING) in perf_sigtrap()
6483 send_sig_perf((void __user *)event->pending_addr, in perf_sigtrap()
6484 event->attr.type, event->attr.sig_data); in perf_sigtrap()
6488 * Deliver the pending work in-event-context or follow the context.
6492 int cpu = READ_ONCE(event->oncpu); in __perf_pending_irq()
6505 if (event->pending_sigtrap) { in __perf_pending_irq()
6506 event->pending_sigtrap = 0; in __perf_pending_irq()
6508 local_dec(&event->ctx->nr_pending); in __perf_pending_irq()
6510 if (event->pending_disable) { in __perf_pending_irq()
6511 event->pending_disable = 0; in __perf_pending_irq()
6518 * CPU-A CPU-B in __perf_pending_irq()
6521 * @pending_disable = CPU-A; in __perf_pending_irq()
6524 * sched-out in __perf_pending_irq()
6525 * @pending_disable = -1; in __perf_pending_irq()
6527 * sched-in in __perf_pending_irq()
6529 * @pending_disable = CPU-B; in __perf_pending_irq()
6535 * But the event runs on CPU-B and wants disabling there. in __perf_pending_irq()
6537 irq_work_queue_on(&event->pending_irq, cpu); in __perf_pending_irq()
6552 * The wakeup isn't bound to the context of the event -- it can happen in perf_pending_irq()
6555 if (event->pending_wakeup) { in perf_pending_irq()
6556 event->pending_wakeup = 0; in perf_pending_irq()
6578 if (event->pending_work) { in perf_pending_task()
6579 event->pending_work = 0; in perf_pending_task()
6581 local_dec(&event->ctx->nr_pending); in perf_pending_task()
6594 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state);
6595 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
6596 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
6604 static_call_update(__perf_guest_state, cbs->state); in perf_register_guest_info_callbacks()
6605 static_call_update(__perf_guest_get_ip, cbs->get_ip); in perf_register_guest_info_callbacks()
6607 /* Implementing ->handle_intel_pt_intr is optional. */ in perf_register_guest_info_callbacks()
6608 if (cbs->handle_intel_pt_intr) in perf_register_guest_info_callbacks()
6610 cbs->handle_intel_pt_intr); in perf_register_guest_info_callbacks()
6649 regs_user->abi = perf_reg_abi(current); in perf_sample_regs_user()
6650 regs_user->regs = regs; in perf_sample_regs_user()
6651 } else if (!(current->flags & PF_KTHREAD)) { in perf_sample_regs_user()
6654 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; in perf_sample_regs_user()
6655 regs_user->regs = NULL; in perf_sample_regs_user()
6662 regs_intr->regs = regs; in perf_sample_regs_intr()
6663 regs_intr->abi = perf_reg_abi(current); in perf_sample_regs_intr()
6681 return TASK_SIZE - addr; in perf_ustack_task_size()
6696 * - TASK_SIZE in perf_sample_ustack_size()
6699 * - remaining sample size in perf_sample_ustack_size()
6716 stack_size = USHRT_MAX - header_size - sizeof(u64); in perf_sample_ustack_size()
6739 * - the size requested by user or the best one we can fit in perf_output_sample_ustack()
6742 * - user stack dump data in perf_output_sample_ustack()
6744 * - the actual dumped size in perf_output_sample_ustack()
6753 dyn_size = dump_size - rem; in perf_output_sample_ustack()
6766 struct perf_event *sampler = event->aux_event; in perf_prepare_sample_aux()
6769 data->aux_size = 0; in perf_prepare_sample_aux()
6774 if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE)) in perf_prepare_sample_aux()
6777 if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id())) in perf_prepare_sample_aux()
6788 if (READ_ONCE(rb->aux_in_sampling)) { in perf_prepare_sample_aux()
6789 data->aux_size = 0; in perf_prepare_sample_aux()
6792 data->aux_size = ALIGN(size, sizeof(u64)); in perf_prepare_sample_aux()
6797 return data->aux_size; in perf_prepare_sample_aux()
6809 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler in perf_pmu_snapshot_aux()
6811 * the IRQ ones, that is, for example, re-starting an event that's just in perf_pmu_snapshot_aux()
6822 WRITE_ONCE(rb->aux_in_sampling, 1); in perf_pmu_snapshot_aux()
6825 ret = event->pmu->snapshot_aux(event, handle, size); in perf_pmu_snapshot_aux()
6828 WRITE_ONCE(rb->aux_in_sampling, 0); in perf_pmu_snapshot_aux()
6838 struct perf_event *sampler = event->aux_event; in perf_aux_sample_output()
6843 if (WARN_ON_ONCE(!sampler || !data->aux_size)) in perf_aux_sample_output()
6850 size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size); in perf_aux_sample_output()
6854 * non-zero surplus that it didn't copy), which in its current in perf_aux_sample_output()
6862 * The pad comes from ALIGN()ing data->aux_size up to u64 in in perf_aux_sample_output()
6865 pad = data->aux_size - size; in perf_aux_sample_output()
6883 data->type = event->attr.sample_type; in __perf_event_header__init_id()
6884 header->size += event->id_header_size; in __perf_event_header__init_id()
6888 data->tid_entry.pid = perf_event_pid(event, current); in __perf_event_header__init_id()
6889 data->tid_entry.tid = perf_event_tid(event, current); in __perf_event_header__init_id()
6893 data->time = perf_event_clock(event); in __perf_event_header__init_id()
6896 data->id = primary_event_id(event); in __perf_event_header__init_id()
6899 data->stream_id = event->id; in __perf_event_header__init_id()
6902 data->cpu_entry.cpu = raw_smp_processor_id(); in __perf_event_header__init_id()
6903 data->cpu_entry.reserved = 0; in __perf_event_header__init_id()
6911 if (event->attr.sample_id_all) in perf_event_header__init_id()
6912 __perf_event_header__init_id(header, data, event, event->attr.sample_type); in perf_event_header__init_id()
6918 u64 sample_type = data->type; in __perf_event__output_id_sample()
6921 perf_output_put(handle, data->tid_entry); in __perf_event__output_id_sample()
6924 perf_output_put(handle, data->time); in __perf_event__output_id_sample()
6927 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
6930 perf_output_put(handle, data->stream_id); in __perf_event__output_id_sample()
6933 perf_output_put(handle, data->cpu_entry); in __perf_event__output_id_sample()
6936 perf_output_put(handle, data->id); in __perf_event__output_id_sample()
6943 if (event->attr.sample_id_all) in perf_event__output_id_sample()
6951 u64 read_format = event->attr.read_format; in perf_output_read_one()
6958 atomic64_read(&event->child_total_time_enabled); in perf_output_read_one()
6962 atomic64_read(&event->child_total_time_running); in perf_output_read_one()
6967 values[n++] = atomic64_read(&event->lost_samples); in perf_output_read_one()
6976 struct perf_event *leader = event->group_leader, *sub; in perf_output_read_group()
6977 u64 read_format = event->attr.read_format; in perf_output_read_group()
6988 values[n++] = 1 + leader->nr_siblings; in perf_output_read_group()
6997 (leader->state == PERF_EVENT_STATE_ACTIVE)) in perf_output_read_group()
6998 leader->pmu->read(leader); in perf_output_read_group()
7004 values[n++] = atomic64_read(&leader->lost_samples); in perf_output_read_group()
7012 (sub->state == PERF_EVENT_STATE_ACTIVE)) in perf_output_read_group()
7013 sub->pmu->read(sub); in perf_output_read_group()
7019 values[n++] = atomic64_read(&sub->lost_samples); in perf_output_read_group()
7034 * child list, not to mention that its impossible to IPI the children running
7041 u64 read_format = event->attr.read_format; in perf_output_read()
7055 if (event->attr.read_format & PERF_FORMAT_GROUP) in perf_output_read()
7066 u64 sample_type = data->type; in perf_output_sample()
7071 perf_output_put(handle, data->id); in perf_output_sample()
7074 perf_output_put(handle, data->ip); in perf_output_sample()
7077 perf_output_put(handle, data->tid_entry); in perf_output_sample()
7080 perf_output_put(handle, data->time); in perf_output_sample()
7083 perf_output_put(handle, data->addr); in perf_output_sample()
7086 perf_output_put(handle, data->id); in perf_output_sample()
7089 perf_output_put(handle, data->stream_id); in perf_output_sample()
7092 perf_output_put(handle, data->cpu_entry); in perf_output_sample()
7095 perf_output_put(handle, data->period); in perf_output_sample()
7103 size += data->callchain->nr; in perf_output_sample()
7105 __output_copy(handle, data->callchain, size); in perf_output_sample()
7109 struct perf_raw_record *raw = data->raw; in perf_output_sample()
7112 struct perf_raw_frag *frag = &raw->frag; in perf_output_sample()
7114 perf_output_put(handle, raw->size); in perf_output_sample()
7116 if (frag->copy) { in perf_output_sample()
7117 __output_custom(handle, frag->copy, in perf_output_sample()
7118 frag->data, frag->size); in perf_output_sample()
7120 __output_copy(handle, frag->data, in perf_output_sample()
7121 frag->size); in perf_output_sample()
7125 frag = frag->next; in perf_output_sample()
7127 if (frag->pad) in perf_output_sample()
7128 __output_skip(handle, NULL, frag->pad); in perf_output_sample()
7142 if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { in perf_output_sample()
7145 size = data->br_stack->nr in perf_output_sample()
7148 perf_output_put(handle, data->br_stack->nr); in perf_output_sample()
7150 perf_output_put(handle, data->br_stack->hw_idx); in perf_output_sample()
7151 perf_output_copy(handle, data->br_stack->entries, size); in perf_output_sample()
7162 u64 abi = data->regs_user.abi; in perf_output_sample()
7171 u64 mask = event->attr.sample_regs_user; in perf_output_sample()
7173 data->regs_user.regs, in perf_output_sample()
7180 data->stack_user_size, in perf_output_sample()
7181 data->regs_user.regs); in perf_output_sample()
7185 perf_output_put(handle, data->weight.full); in perf_output_sample()
7188 perf_output_put(handle, data->data_src.val); in perf_output_sample()
7191 perf_output_put(handle, data->txn); in perf_output_sample()
7194 u64 abi = data->regs_intr.abi; in perf_output_sample()
7202 u64 mask = event->attr.sample_regs_intr; in perf_output_sample()
7205 data->regs_intr.regs, in perf_output_sample()
7211 perf_output_put(handle, data->phys_addr); in perf_output_sample()
7214 perf_output_put(handle, data->cgroup); in perf_output_sample()
7217 perf_output_put(handle, data->data_page_size); in perf_output_sample()
7220 perf_output_put(handle, data->code_page_size); in perf_output_sample()
7223 perf_output_put(handle, data->aux_size); in perf_output_sample()
7225 if (data->aux_size) in perf_output_sample()
7229 if (!event->attr.watermark) { in perf_output_sample()
7230 int wakeup_events = event->attr.wakeup_events; in perf_output_sample()
7233 struct perf_buffer *rb = handle->rb; in perf_output_sample()
7234 int events = local_inc_return(&rb->events); in perf_output_sample()
7237 local_sub(wakeup_events, &rb->events); in perf_output_sample()
7238 local_inc(&rb->wakeup); in perf_output_sample()
7261 * Try IRQ-safe get_user_page_fast_only first. in perf_virt_to_phys()
7264 if (current->mm != NULL) { in perf_virt_to_phys()
7345 * Software page-table walkers must disable IRQs, in perf_get_page_size()
7350 mm = current->mm; in perf_get_page_size()
7371 bool kernel = !event->attr.exclude_callchain_kernel; in perf_callchain()
7372 bool user = !event->attr.exclude_callchain_user; in perf_callchain()
7373 /* Disallow cross-task user callchains. */ in perf_callchain()
7374 bool crosstask = event->ctx->task && event->ctx->task != current; in perf_callchain()
7375 const u32 max_stack = event->attr.sample_max_stack; in perf_callchain()
7391 u64 sample_type = event->attr.sample_type; in perf_prepare_sample()
7394 header->type = PERF_RECORD_SAMPLE; in perf_prepare_sample()
7395 header->size = sizeof(*header) + event->header_size; in perf_prepare_sample()
7397 header->misc = 0; in perf_prepare_sample()
7398 header->misc |= perf_misc_flags(regs); in perf_prepare_sample()
7404 filtered_sample_type = sample_type & ~data->sample_flags; in perf_prepare_sample()
7408 data->ip = perf_instruction_pointer(regs); in perf_prepare_sample()
7414 data->callchain = perf_callchain(event, regs); in perf_prepare_sample()
7416 size += data->callchain->nr; in perf_prepare_sample()
7418 header->size += size * sizeof(u64); in perf_prepare_sample()
7422 struct perf_raw_record *raw = data->raw; in perf_prepare_sample()
7425 if (raw && (data->sample_flags & PERF_SAMPLE_RAW)) { in perf_prepare_sample()
7426 struct perf_raw_frag *frag = &raw->frag; in perf_prepare_sample()
7430 sum += frag->size; in perf_prepare_sample()
7433 frag = frag->next; in perf_prepare_sample()
7437 raw->size = size - sizeof(u32); in perf_prepare_sample()
7438 frag->pad = raw->size - sum; in perf_prepare_sample()
7441 data->raw = NULL; in perf_prepare_sample()
7444 header->size += size; in perf_prepare_sample()
7449 if (data->sample_flags & PERF_SAMPLE_BRANCH_STACK) { in perf_prepare_sample()
7453 size += data->br_stack->nr in perf_prepare_sample()
7456 header->size += size; in perf_prepare_sample()
7460 perf_sample_regs_user(&data->regs_user, regs); in perf_prepare_sample()
7466 if (data->regs_user.regs) { in perf_prepare_sample()
7467 u64 mask = event->attr.sample_regs_user; in perf_prepare_sample()
7471 header->size += size; in perf_prepare_sample()
7481 u16 stack_size = event->attr.sample_stack_user; in perf_prepare_sample()
7484 stack_size = perf_sample_ustack_size(stack_size, header->size, in perf_prepare_sample()
7485 data->regs_user.regs); in perf_prepare_sample()
7495 data->stack_user_size = stack_size; in perf_prepare_sample()
7496 header->size += size; in perf_prepare_sample()
7500 data->weight.full = 0; in perf_prepare_sample()
7503 data->data_src.val = PERF_MEM_NA; in perf_prepare_sample()
7506 data->txn = 0; in perf_prepare_sample()
7510 data->addr = 0; in perf_prepare_sample()
7517 perf_sample_regs_intr(&data->regs_intr, regs); in perf_prepare_sample()
7519 if (data->regs_intr.regs) { in perf_prepare_sample()
7520 u64 mask = event->attr.sample_regs_intr; in perf_prepare_sample()
7525 header->size += size; in perf_prepare_sample()
7530 data->phys_addr = perf_virt_to_phys(data->addr); in perf_prepare_sample()
7537 cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup; in perf_prepare_sample()
7538 data->cgroup = cgroup_id(cgrp); in perf_prepare_sample()
7544 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, in perf_prepare_sample()
7548 data->data_page_size = perf_get_page_size(data->addr); in perf_prepare_sample()
7551 data->code_page_size = perf_get_page_size(data->ip); in perf_prepare_sample()
7556 header->size += sizeof(u64); /* size */ in perf_prepare_sample()
7564 size = min_t(size_t, U16_MAX - header->size, in perf_prepare_sample()
7565 event->attr.aux_sample_size); in perf_prepare_sample()
7569 WARN_ON_ONCE(size + header->size > U16_MAX); in perf_prepare_sample()
7570 header->size += size; in perf_prepare_sample()
7580 WARN_ON_ONCE(header->size & 7); in perf_prepare_sample()
7659 .size = sizeof(read_event) + event->read_size, in perf_event_read_event()
7687 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_iterate_ctx()
7689 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_ctx()
7704 list_for_each_entry_rcu(event, &pel->list, sb_list) { in perf_iterate_sb_cpu()
7707 * if we observe event->ctx, both event and ctx will be in perf_iterate_sb_cpu()
7710 if (!smp_load_acquire(&event->ctx)) in perf_iterate_sb_cpu()
7713 if (event->state < PERF_EVENT_STATE_INACTIVE) in perf_iterate_sb_cpu()
7722 * Iterate all events that need to receive side-band events.
7750 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); in perf_iterate_sb()
7760 * Clear all file-based filters at exec, they'll have to be
7761 * re-instated when/if these objects are mmapped again.
7773 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_exec()
7774 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_exec()
7775 if (filter->path.dentry) { in perf_event_addr_filters_exec()
7776 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_exec()
7777 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_exec()
7785 event->addr_filters_gen++; in perf_event_addr_filters_exec()
7786 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_exec()
7802 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); in perf_event_exec()
7818 struct perf_event *parent = event->parent; in __perf_event_output_stop()
7820 struct perf_buffer *rb = ro->rb; in __perf_event_output_stop()
7833 * ring-buffer, but it will be the child that's actually using it. in __perf_event_output_stop()
7838 * So ring_buffer_attach() has to stop an aux event before re-assigning in __perf_event_output_stop()
7841 if (rcu_dereference(parent->rb) == rb) in __perf_event_output_stop()
7842 ro->err = __perf_event_stop(&sd); in __perf_event_output_stop()
7848 struct pmu *pmu = event->ctx->pmu; in __perf_pmu_output_stop()
7849 struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); in __perf_pmu_output_stop()
7851 .rb = event->rb, in __perf_pmu_output_stop()
7855 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); in __perf_pmu_output_stop()
7856 if (cpuctx->task_ctx) in __perf_pmu_output_stop()
7857 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, in __perf_pmu_output_stop()
7871 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { in perf_pmu_output_stop()
7873 * For per-CPU events, we need to make sure that neither they in perf_pmu_output_stop()
7874 * nor their children are running; for cpu==-1 events it's in perf_pmu_output_stop()
7878 cpu = iter->cpu; in perf_pmu_output_stop()
7879 if (cpu == -1) in perf_pmu_output_stop()
7880 cpu = READ_ONCE(iter->oncpu); in perf_pmu_output_stop()
7882 if (cpu == -1) in perf_pmu_output_stop()
7886 if (err == -EAGAIN) { in perf_pmu_output_stop()
7895 * task tracking -- fork/exit
7917 return event->attr.comm || event->attr.mmap || in perf_event_task_match()
7918 event->attr.mmap2 || event->attr.mmap_data || in perf_event_task_match()
7919 event->attr.task; in perf_event_task_match()
7928 struct task_struct *task = task_event->task; in perf_event_task_output()
7929 int ret, size = task_event->event_id.header.size; in perf_event_task_output()
7934 perf_event_header__init_id(&task_event->event_id.header, &sample, event); in perf_event_task_output()
7937 task_event->event_id.header.size); in perf_event_task_output()
7941 task_event->event_id.pid = perf_event_pid(event, task); in perf_event_task_output()
7942 task_event->event_id.tid = perf_event_tid(event, task); in perf_event_task_output()
7944 if (task_event->event_id.header.type == PERF_RECORD_EXIT) { in perf_event_task_output()
7945 task_event->event_id.ppid = perf_event_pid(event, in perf_event_task_output()
7946 task->real_parent); in perf_event_task_output()
7947 task_event->event_id.ptid = perf_event_pid(event, in perf_event_task_output()
7948 task->real_parent); in perf_event_task_output()
7950 task_event->event_id.ppid = perf_event_pid(event, current); in perf_event_task_output()
7951 task_event->event_id.ptid = perf_event_tid(event, current); in perf_event_task_output()
7954 task_event->event_id.time = perf_event_clock(event); in perf_event_task_output()
7956 perf_output_put(&handle, task_event->event_id); in perf_event_task_output()
7962 task_event->event_id.header.size = size; in perf_event_task_output()
8023 return event->attr.comm; in perf_event_comm_match()
8032 int size = comm_event->event_id.header.size; in perf_event_comm_output()
8038 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); in perf_event_comm_output()
8040 comm_event->event_id.header.size); in perf_event_comm_output()
8045 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); in perf_event_comm_output()
8046 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); in perf_event_comm_output()
8048 perf_output_put(&handle, comm_event->event_id); in perf_event_comm_output()
8049 __output_copy(&handle, comm_event->comm, in perf_event_comm_output()
8050 comm_event->comm_size); in perf_event_comm_output()
8056 comm_event->event_id.header.size = size; in perf_event_comm_output()
8065 strlcpy(comm, comm_event->task->comm, sizeof(comm)); in perf_event_comm_event()
8068 comm_event->comm = comm; in perf_event_comm_event()
8069 comm_event->comm_size = size; in perf_event_comm_event()
8071 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; in perf_event_comm_event()
8122 return event->attr.namespaces; in perf_event_namespaces_match()
8131 u16 header_size = namespaces_event->event_id.header.size; in perf_event_namespaces_output()
8137 perf_event_header__init_id(&namespaces_event->event_id.header, in perf_event_namespaces_output()
8140 namespaces_event->event_id.header.size); in perf_event_namespaces_output()
8144 namespaces_event->event_id.pid = perf_event_pid(event, in perf_event_namespaces_output()
8145 namespaces_event->task); in perf_event_namespaces_output()
8146 namespaces_event->event_id.tid = perf_event_tid(event, in perf_event_namespaces_output()
8147 namespaces_event->task); in perf_event_namespaces_output()
8149 perf_output_put(&handle, namespaces_event->event_id); in perf_event_namespaces_output()
8155 namespaces_event->event_id.header.size = header_size; in perf_event_namespaces_output()
8168 ns_inode = ns_path.dentry->d_inode; in perf_fill_ns_link_info()
8169 ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev); in perf_fill_ns_link_info()
8170 ns_link_info->ino = ns_inode->i_ino; in perf_fill_ns_link_info()
8250 return event->attr.cgroup; in perf_event_cgroup_match()
8258 u16 header_size = cgroup_event->event_id.header.size; in perf_event_cgroup_output()
8264 perf_event_header__init_id(&cgroup_event->event_id.header, in perf_event_cgroup_output()
8267 cgroup_event->event_id.header.size); in perf_event_cgroup_output()
8271 perf_output_put(&handle, cgroup_event->event_id); in perf_event_cgroup_output()
8272 __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); in perf_event_cgroup_output()
8278 cgroup_event->event_id.header.size = header_size; in perf_event_cgroup_output()
8307 cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64)); in perf_event_cgroup()
8363 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_match()
8364 int executable = vma->vm_flags & VM_EXEC; in perf_event_mmap_match()
8366 return (!executable && event->attr.mmap_data) || in perf_event_mmap_match()
8367 (executable && (event->attr.mmap || event->attr.mmap2)); in perf_event_mmap_match()
8376 int size = mmap_event->event_id.header.size; in perf_event_mmap_output()
8377 u32 type = mmap_event->event_id.header.type; in perf_event_mmap_output()
8384 if (event->attr.mmap2) { in perf_event_mmap_output()
8385 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; in perf_event_mmap_output()
8386 mmap_event->event_id.header.size += sizeof(mmap_event->maj); in perf_event_mmap_output()
8387 mmap_event->event_id.header.size += sizeof(mmap_event->min); in perf_event_mmap_output()
8388 mmap_event->event_id.header.size += sizeof(mmap_event->ino); in perf_event_mmap_output()
8389 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); in perf_event_mmap_output()
8390 mmap_event->event_id.header.size += sizeof(mmap_event->prot); in perf_event_mmap_output()
8391 mmap_event->event_id.header.size += sizeof(mmap_event->flags); in perf_event_mmap_output()
8394 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); in perf_event_mmap_output()
8396 mmap_event->event_id.header.size); in perf_event_mmap_output()
8400 mmap_event->event_id.pid = perf_event_pid(event, current); in perf_event_mmap_output()
8401 mmap_event->event_id.tid = perf_event_tid(event, current); in perf_event_mmap_output()
8403 use_build_id = event->attr.build_id && mmap_event->build_id_size; in perf_event_mmap_output()
8405 if (event->attr.mmap2 && use_build_id) in perf_event_mmap_output()
8406 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; in perf_event_mmap_output()
8408 perf_output_put(&handle, mmap_event->event_id); in perf_event_mmap_output()
8410 if (event->attr.mmap2) { in perf_event_mmap_output()
8412 u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 }; in perf_event_mmap_output()
8415 __output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX); in perf_event_mmap_output()
8417 perf_output_put(&handle, mmap_event->maj); in perf_event_mmap_output()
8418 perf_output_put(&handle, mmap_event->min); in perf_event_mmap_output()
8419 perf_output_put(&handle, mmap_event->ino); in perf_event_mmap_output()
8420 perf_output_put(&handle, mmap_event->ino_generation); in perf_event_mmap_output()
8422 perf_output_put(&handle, mmap_event->prot); in perf_event_mmap_output()
8423 perf_output_put(&handle, mmap_event->flags); in perf_event_mmap_output()
8426 __output_copy(&handle, mmap_event->file_name, in perf_event_mmap_output()
8427 mmap_event->file_size); in perf_event_mmap_output()
8433 mmap_event->event_id.header.size = size; in perf_event_mmap_output()
8434 mmap_event->event_id.header.type = type; in perf_event_mmap_output()
8439 struct vm_area_struct *vma = mmap_event->vma; in perf_event_mmap_event()
8440 struct file *file = vma->vm_file; in perf_event_mmap_event()
8449 if (vma->vm_flags & VM_READ) in perf_event_mmap_event()
8451 if (vma->vm_flags & VM_WRITE) in perf_event_mmap_event()
8453 if (vma->vm_flags & VM_EXEC) in perf_event_mmap_event()
8456 if (vma->vm_flags & VM_MAYSHARE) in perf_event_mmap_event()
8461 if (vma->vm_flags & VM_LOCKED) in perf_event_mmap_event()
8480 name = file_path(file, buf, PATH_MAX - sizeof(u64)); in perf_event_mmap_event()
8485 inode = file_inode(vma->vm_file); in perf_event_mmap_event()
8486 dev = inode->i_sb->s_dev; in perf_event_mmap_event()
8487 ino = inode->i_ino; in perf_event_mmap_event()
8488 gen = inode->i_generation; in perf_event_mmap_event()
8494 if (vma->vm_ops && vma->vm_ops->name) { in perf_event_mmap_event()
8495 name = (char *) vma->vm_ops->name(vma); in perf_event_mmap_event()
8504 if (vma->vm_start <= vma->vm_mm->start_brk && in perf_event_mmap_event()
8505 vma->vm_end >= vma->vm_mm->brk) { in perf_event_mmap_event()
8509 if (vma->vm_start <= vma->vm_mm->start_stack && in perf_event_mmap_event()
8510 vma->vm_end >= vma->vm_mm->start_stack) { in perf_event_mmap_event()
8532 mmap_event->file_name = name; in perf_event_mmap_event()
8533 mmap_event->file_size = size; in perf_event_mmap_event()
8534 mmap_event->maj = maj; in perf_event_mmap_event()
8535 mmap_event->min = min; in perf_event_mmap_event()
8536 mmap_event->ino = ino; in perf_event_mmap_event()
8537 mmap_event->ino_generation = gen; in perf_event_mmap_event()
8538 mmap_event->prot = prot; in perf_event_mmap_event()
8539 mmap_event->flags = flags; in perf_event_mmap_event()
8541 if (!(vma->vm_flags & VM_EXEC)) in perf_event_mmap_event()
8542 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; in perf_event_mmap_event()
8544 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; in perf_event_mmap_event()
8547 build_id_parse(vma, mmap_event->build_id, &mmap_event->build_id_size); in perf_event_mmap_event()
8563 /* d_inode(NULL) won't be equal to any mapped user-space file */ in perf_addr_filter_match()
8564 if (!filter->path.dentry) in perf_addr_filter_match()
8567 if (d_inode(filter->path.dentry) != file_inode(file)) in perf_addr_filter_match()
8570 if (filter->offset > offset + size) in perf_addr_filter_match()
8573 if (filter->offset + filter->size < offset) in perf_addr_filter_match()
8583 unsigned long vma_size = vma->vm_end - vma->vm_start; in perf_addr_filter_vma_adjust()
8584 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; in perf_addr_filter_vma_adjust()
8585 struct file *file = vma->vm_file; in perf_addr_filter_vma_adjust()
8590 if (filter->offset < off) { in perf_addr_filter_vma_adjust()
8591 fr->start = vma->vm_start; in perf_addr_filter_vma_adjust()
8592 fr->size = min(vma_size, filter->size - (off - filter->offset)); in perf_addr_filter_vma_adjust()
8594 fr->start = vma->vm_start + filter->offset - off; in perf_addr_filter_vma_adjust()
8595 fr->size = min(vma->vm_end - fr->start, filter->size); in perf_addr_filter_vma_adjust()
8612 if (!vma->vm_file) in __perf_addr_filters_adjust()
8615 raw_spin_lock_irqsave(&ifh->lock, flags); in __perf_addr_filters_adjust()
8616 list_for_each_entry(filter, &ifh->list, entry) { in __perf_addr_filters_adjust()
8618 &event->addr_filter_ranges[count])) in __perf_addr_filters_adjust()
8625 event->addr_filters_gen++; in __perf_addr_filters_adjust()
8626 raw_spin_unlock_irqrestore(&ifh->lock, flags); in __perf_addr_filters_adjust()
8644 if (!(vma->vm_flags & VM_EXEC)) in perf_addr_filters_adjust()
8649 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); in perf_addr_filters_adjust()
8677 .start = vma->vm_start, in perf_event_mmap()
8678 .len = vma->vm_end - vma->vm_start, in perf_event_mmap()
8679 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, in perf_event_mmap()
8777 return event->attr.context_switch; in perf_event_switch_match()
8790 /* Only CPU-wide events are allowed to see next/prev pid/tid */ in perf_event_switch_output()
8791 if (event->ctx->task) { in perf_event_switch_output()
8792 se->event_id.header.type = PERF_RECORD_SWITCH; in perf_event_switch_output()
8793 se->event_id.header.size = sizeof(se->event_id.header); in perf_event_switch_output()
8795 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; in perf_event_switch_output()
8796 se->event_id.header.size = sizeof(se->event_id); in perf_event_switch_output()
8797 se->event_id.next_prev_pid = in perf_event_switch_output()
8798 perf_event_pid(event, se->next_prev); in perf_event_switch_output()
8799 se->event_id.next_prev_tid = in perf_event_switch_output()
8800 perf_event_tid(event, se->next_prev); in perf_event_switch_output()
8803 perf_event_header__init_id(&se->event_id.header, &sample, event); in perf_event_switch_output()
8805 ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); in perf_event_switch_output()
8809 if (event->ctx->task) in perf_event_switch_output()
8810 perf_output_put(&handle, se->event_id.header); in perf_event_switch_output()
8812 perf_output_put(&handle, se->event_id); in perf_event_switch_output()
8840 if (!sched_in && task->on_rq) { in perf_event_switch()
8871 .stream_id = event->id, in perf_log_throttle()
8907 return event->attr.ksymbol; in perf_event_ksymbol_match()
8920 perf_event_header__init_id(&ksymbol_event->event_id.header, in perf_event_ksymbol_output()
8923 ksymbol_event->event_id.header.size); in perf_event_ksymbol_output()
8927 perf_output_put(&handle, ksymbol_event->event_id); in perf_event_ksymbol_output()
8928 __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len); in perf_event_ksymbol_output()
8997 return event->attr.bpf_event; in perf_event_bpf_match()
9010 perf_event_header__init_id(&bpf_event->event_id.header, in perf_event_bpf_output()
9013 bpf_event->event_id.header.size); in perf_event_bpf_output()
9017 perf_output_put(&handle, bpf_event->event_id); in perf_event_bpf_output()
9029 if (prog->aux->func_cnt == 0) { in perf_event_bpf_emit_ksymbols()
9031 (u64)(unsigned long)prog->bpf_func, in perf_event_bpf_emit_ksymbols()
9032 prog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9033 prog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9035 for (i = 0; i < prog->aux->func_cnt; i++) { in perf_event_bpf_emit_ksymbols()
9036 struct bpf_prog *subprog = prog->aux->func[i]; in perf_event_bpf_emit_ksymbols()
9040 (u64)(unsigned long)subprog->bpf_func, in perf_event_bpf_emit_ksymbols()
9041 subprog->jited_len, unregister, in perf_event_bpf_emit_ksymbols()
9042 subprog->aux->ksym.name); in perf_event_bpf_emit_ksymbols()
9079 .id = prog->aux->id, in perf_event_bpf_event()
9085 memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE); in perf_event_bpf_event()
9105 return event->attr.text_poke; in perf_event_text_poke_match()
9119 perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event); in perf_event_text_poke_output()
9122 text_poke_event->event_id.header.size); in perf_event_text_poke_output()
9126 perf_output_put(&handle, text_poke_event->event_id); in perf_event_text_poke_output()
9127 perf_output_put(&handle, text_poke_event->old_len); in perf_event_text_poke_output()
9128 perf_output_put(&handle, text_poke_event->new_len); in perf_event_text_poke_output()
9130 __output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len); in perf_event_text_poke_output()
9131 __output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len); in perf_event_text_poke_output()
9133 if (text_poke_event->pad) in perf_event_text_poke_output()
9134 __output_copy(&handle, &padding, text_poke_event->pad); in perf_event_text_poke_output()
9152 pad = ALIGN(tot, sizeof(u64)) - tot; in perf_event_text_poke()
9175 event->attach_state |= PERF_ATTACH_ITRACE; in perf_event_itrace_started()
9189 if (event->parent) in perf_log_itrace_start()
9190 event = event->parent; in perf_log_itrace_start()
9192 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || in perf_log_itrace_start()
9193 event->attach_state & PERF_ATTACH_ITRACE) in perf_log_itrace_start()
9224 if (event->parent) in perf_report_aux_output_id()
9225 event = event->parent; in perf_report_aux_output_id()
9247 struct hw_perf_event *hwc = &event->hw; in __perf_event_account_interrupt()
9252 if (seq != hwc->interrupts_seq) { in __perf_event_account_interrupt()
9253 hwc->interrupts_seq = seq; in __perf_event_account_interrupt()
9254 hwc->interrupts = 1; in __perf_event_account_interrupt()
9256 hwc->interrupts++; in __perf_event_account_interrupt()
9258 && hwc->interrupts >= max_samples_per_tick)) { in __perf_event_account_interrupt()
9261 hwc->interrupts = MAX_INTERRUPTS; in __perf_event_account_interrupt()
9267 if (event->attr.freq) { in __perf_event_account_interrupt()
9269 s64 delta = now - hwc->freq_time_stamp; in __perf_event_account_interrupt()
9271 hwc->freq_time_stamp = now; in __perf_event_account_interrupt()
9274 perf_adjust_period(event, delta, hwc->last_period, true); in __perf_event_account_interrupt()
9292 if (event->attr.exclude_kernel && !user_mode(regs)) in sample_is_allowed()
9306 int events = atomic_read(&event->event_limit); in __perf_event_overflow()
9310 * Non-sampling counters might still use the PMI to fold short in __perf_event_overflow()
9323 event->pending_kill = POLL_IN; in __perf_event_overflow()
9324 if (events && atomic_dec_and_test(&event->event_limit)) { in __perf_event_overflow()
9326 event->pending_kill = POLL_HUP; in __perf_event_overflow()
9330 if (event->attr.sigtrap) { in __perf_event_overflow()
9342 if (!event->pending_sigtrap) { in __perf_event_overflow()
9343 event->pending_sigtrap = pending_id; in __perf_event_overflow()
9344 local_inc(&event->ctx->nr_pending); in __perf_event_overflow()
9345 } else if (event->attr.exclude_kernel && valid_sample) { in __perf_event_overflow()
9353 * 2. Events that can overflow again before the IRQ- in __perf_event_overflow()
9356 * check 32-bit hash of the current IP. in __perf_event_overflow()
9358 WARN_ON_ONCE(event->pending_sigtrap != pending_id); in __perf_event_overflow()
9361 event->pending_addr = 0; in __perf_event_overflow()
9362 if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR)) in __perf_event_overflow()
9363 event->pending_addr = data->addr; in __perf_event_overflow()
9364 irq_work_queue(&event->pending_irq); in __perf_event_overflow()
9367 READ_ONCE(event->overflow_handler)(event, data, regs); in __perf_event_overflow()
9369 if (*perf_event_fasync(event) && event->pending_kill) { in __perf_event_overflow()
9370 event->pending_wakeup = 1; in __perf_event_overflow()
9371 irq_work_queue(&event->pending_irq); in __perf_event_overflow()
9400 * We directly increment event->count and keep a second value in
9401 * event->hw.period_left to count intervals. This period event
9402 * is kept in the range [-sample_period, 0] so that we can use the
9408 struct hw_perf_event *hwc = &event->hw; in perf_swevent_set_period()
9409 u64 period = hwc->last_period; in perf_swevent_set_period()
9413 hwc->last_period = hwc->sample_period; in perf_swevent_set_period()
9416 old = val = local64_read(&hwc->period_left); in perf_swevent_set_period()
9422 val -= offset; in perf_swevent_set_period()
9423 if (local64_cmpxchg(&hwc->period_left, old, val) != old) in perf_swevent_set_period()
9433 struct hw_perf_event *hwc = &event->hw; in perf_swevent_overflow()
9439 if (hwc->interrupts == MAX_INTERRUPTS) in perf_swevent_overflow()
9442 for (; overflow; overflow--) { in perf_swevent_overflow()
9447 * hwc->interrupts == MAX_INTERRUPTS. in perf_swevent_overflow()
9459 struct hw_perf_event *hwc = &event->hw; in perf_swevent_event()
9461 local64_add(nr, &event->count); in perf_swevent_event()
9469 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { in perf_swevent_event()
9470 data->period = nr; in perf_swevent_event()
9473 data->period = event->hw.last_period; in perf_swevent_event()
9475 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) in perf_swevent_event()
9478 if (local64_add_negative(nr, &hwc->period_left)) in perf_swevent_event()
9487 if (event->hw.state & PERF_HES_STOPPED) in perf_exclude_event()
9491 if (event->attr.exclude_user && user_mode(regs)) in perf_exclude_event()
9494 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_exclude_event()
9507 if (event->attr.type != type) in perf_swevent_match()
9510 if (event->attr.config != event_id) in perf_swevent_match()
9531 return &hlist->heads[hash]; in __find_swevent_head()
9540 hlist = rcu_dereference(swhash->swevent_hlist); in find_swevent_head_rcu()
9552 u32 event_id = event->attr.config; in find_swevent_head()
9553 u64 type = event->attr.type; in find_swevent_head()
9560 hlist = rcu_dereference_protected(swhash->swevent_hlist, in find_swevent_head()
9561 lockdep_is_held(&event->ctx->lock)); in find_swevent_head()
9596 return get_recursion_context(swhash->recursion); in perf_swevent_get_recursion_context()
9604 put_recursion_context(swhash->recursion, rctx); in perf_swevent_put_recursion_context()
9641 struct hw_perf_event *hwc = &event->hw; in perf_swevent_add()
9645 hwc->last_period = hwc->sample_period; in perf_swevent_add()
9649 hwc->state = !(flags & PERF_EF_START); in perf_swevent_add()
9653 return -EINVAL; in perf_swevent_add()
9655 hlist_add_head_rcu(&event->hlist_entry, head); in perf_swevent_add()
9663 hlist_del_rcu(&event->hlist_entry); in perf_swevent_del()
9668 event->hw.state = 0; in perf_swevent_start()
9673 event->hw.state = PERF_HES_STOPPED; in perf_swevent_stop()
9680 return rcu_dereference_protected(swhash->swevent_hlist, in swevent_hlist_deref()
9681 lockdep_is_held(&swhash->hlist_mutex)); in swevent_hlist_deref()
9691 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); in swevent_hlist_release()
9699 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
9701 if (!--swhash->hlist_refcount) in swevent_hlist_put_cpu()
9704 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_put_cpu()
9720 mutex_lock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
9727 err = -ENOMEM; in swevent_hlist_get_cpu()
9730 rcu_assign_pointer(swhash->swevent_hlist, hlist); in swevent_hlist_get_cpu()
9732 swhash->hlist_refcount++; in swevent_hlist_get_cpu()
9734 mutex_unlock(&swhash->hlist_mutex); in swevent_hlist_get_cpu()
9767 u64 event_id = event->attr.config; in sw_perf_event_destroy()
9769 WARN_ON(event->parent); in sw_perf_event_destroy()
9777 u64 event_id = event->attr.config; in perf_swevent_init()
9779 if (event->attr.type != PERF_TYPE_SOFTWARE) in perf_swevent_init()
9780 return -ENOENT; in perf_swevent_init()
9786 return -EOPNOTSUPP; in perf_swevent_init()
9791 return -ENOENT; in perf_swevent_init()
9798 return -ENOENT; in perf_swevent_init()
9800 if (!event->parent) { in perf_swevent_init()
9808 event->destroy = sw_perf_event_destroy; in perf_swevent_init()
9832 void *record = data->raw->frag.data; in perf_tp_filter_match()
9835 if (event->parent) in perf_tp_filter_match()
9836 event = event->parent; in perf_tp_filter_match()
9838 if (likely(!event->filter) || filter_match_preds(event->filter, record)) in perf_tp_filter_match()
9847 if (event->hw.state & PERF_HES_STOPPED) in perf_tp_event_match()
9850 * If exclude_kernel, only trace user-space tracepoints (uprobes) in perf_tp_event_match()
9852 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_tp_event_match()
9873 perf_tp_event(call->event.type, count, raw_data, size, regs, head, in perf_trace_run_bpf_submit()
9912 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); in perf_tp_event()
9916 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { in perf_tp_event()
9917 if (event->cpu != smp_processor_id()) in perf_tp_event()
9919 if (event->attr.type != PERF_TYPE_TRACEPOINT) in perf_tp_event()
9921 if (event->attr.config != entry->type) in perf_tp_event()
9924 if (event->attr.sigtrap) in perf_tp_event()
9946 if (event->attr.type != PERF_TYPE_TRACEPOINT) in perf_tp_event_init()
9947 return -ENOENT; in perf_tp_event_init()
9953 return -EOPNOTSUPP; in perf_tp_event_init()
9959 event->destroy = tp_perf_event_destroy; in perf_tp_event_init()
9993 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS,
10032 if (event->attr.type != perf_kprobe.type) in perf_kprobe_event_init()
10033 return -ENOENT; in perf_kprobe_event_init()
10036 return -EACCES; in perf_kprobe_event_init()
10042 return -EOPNOTSUPP; in perf_kprobe_event_init()
10044 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_kprobe_event_init()
10049 event->destroy = perf_kprobe_destroy; in perf_kprobe_event_init()
10056 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63");
10092 if (event->attr.type != perf_uprobe.type) in perf_uprobe_event_init()
10093 return -ENOENT; in perf_uprobe_event_init()
10096 return -EACCES; in perf_uprobe_event_init()
10102 return -EOPNOTSUPP; in perf_uprobe_event_init()
10104 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; in perf_uprobe_event_init()
10105 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT; in perf_uprobe_event_init()
10110 event->destroy = perf_uprobe_destroy; in perf_uprobe_event_init()
10120 perf_pmu_register(&perf_kprobe, "kprobe", -1); in perf_tp_register()
10123 perf_pmu_register(&perf_uprobe, "uprobe", -1); in perf_tp_register()
10148 prog = READ_ONCE(event->prog); in bpf_overflow_handler()
10150 if (prog->call_get_stack && in bpf_overflow_handler()
10151 (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) && in bpf_overflow_handler()
10152 !(data->sample_flags & PERF_SAMPLE_CALLCHAIN)) { in bpf_overflow_handler()
10153 data->callchain = perf_callchain(event, regs); in bpf_overflow_handler()
10154 data->sample_flags |= PERF_SAMPLE_CALLCHAIN; in bpf_overflow_handler()
10165 event->orig_overflow_handler(event, data, regs); in bpf_overflow_handler()
10172 if (event->overflow_handler_context) in perf_event_set_bpf_handler()
10174 return -EINVAL; in perf_event_set_bpf_handler()
10176 if (event->prog) in perf_event_set_bpf_handler()
10177 return -EEXIST; in perf_event_set_bpf_handler()
10179 if (prog->type != BPF_PROG_TYPE_PERF_EVENT) in perf_event_set_bpf_handler()
10180 return -EINVAL; in perf_event_set_bpf_handler()
10182 if (event->attr.precise_ip && in perf_event_set_bpf_handler()
10183 prog->call_get_stack && in perf_event_set_bpf_handler()
10184 (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) || in perf_event_set_bpf_handler()
10185 event->attr.exclude_callchain_kernel || in perf_event_set_bpf_handler()
10186 event->attr.exclude_callchain_user)) { in perf_event_set_bpf_handler()
10196 return -EPROTO; in perf_event_set_bpf_handler()
10199 event->prog = prog; in perf_event_set_bpf_handler()
10200 event->bpf_cookie = bpf_cookie; in perf_event_set_bpf_handler()
10201 event->orig_overflow_handler = READ_ONCE(event->overflow_handler); in perf_event_set_bpf_handler()
10202 WRITE_ONCE(event->overflow_handler, bpf_overflow_handler); in perf_event_set_bpf_handler()
10208 struct bpf_prog *prog = event->prog; in perf_event_free_bpf_handler()
10213 WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler); in perf_event_free_bpf_handler()
10214 event->prog = NULL; in perf_event_free_bpf_handler()
10222 return -EOPNOTSUPP; in perf_event_set_bpf_handler()
10235 if (event->pmu == &perf_tracepoint) in perf_event_is_tracing()
10238 if (event->pmu == &perf_kprobe) in perf_event_is_tracing()
10242 if (event->pmu == &perf_uprobe) in perf_event_is_tracing()
10256 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE; in perf_event_set_bpf_prog()
10257 is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE; in perf_event_set_bpf_prog()
10258 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; in perf_event_set_bpf_prog()
10259 is_syscall_tp = is_syscall_trace_event(event->tp_event); in perf_event_set_bpf_prog()
10262 return -EINVAL; in perf_event_set_bpf_prog()
10264 if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) || in perf_event_set_bpf_prog()
10265 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) || in perf_event_set_bpf_prog()
10266 (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) in perf_event_set_bpf_prog()
10267 return -EINVAL; in perf_event_set_bpf_prog()
10269 if (prog->type == BPF_PROG_TYPE_KPROBE && prog->aux->sleepable && !is_uprobe) in perf_event_set_bpf_prog()
10271 return -EINVAL; in perf_event_set_bpf_prog()
10274 if (prog->kprobe_override && !is_kprobe) in perf_event_set_bpf_prog()
10275 return -EINVAL; in perf_event_set_bpf_prog()
10278 int off = trace_event_get_offsets(event->tp_event); in perf_event_set_bpf_prog()
10280 if (prog->aux->max_ctx_offset > off) in perf_event_set_bpf_prog()
10281 return -EACCES; in perf_event_set_bpf_prog()
10309 return -ENOENT; in perf_event_set_bpf_prog()
10323 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); in perf_bp_event()
10325 if (!bp->hw.state && !perf_exclude_event(bp, regs)) in perf_bp_event()
10336 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); in perf_addr_filter_new()
10343 INIT_LIST_HEAD(&filter->entry); in perf_addr_filter_new()
10344 list_add_tail(&filter->entry, filters); in perf_addr_filter_new()
10354 path_put(&filter->path); in free_filters_list()
10355 list_del(&filter->entry); in free_filters_list()
10373 if (event->parent) in perf_addr_filters_splice()
10376 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
10378 list_splice_init(&event->addr_filters.list, &list); in perf_addr_filters_splice()
10380 list_splice(head, &event->addr_filters.list); in perf_addr_filters_splice()
10382 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); in perf_addr_filters_splice()
10400 if (!vma->vm_file) in perf_addr_filter_apply()
10415 struct task_struct *task = READ_ONCE(event->ctx->task); in perf_event_addr_filters_apply()
10422 * We may observe TASK_TOMBSTONE, which means that the event tear-down in perf_event_addr_filters_apply()
10428 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
10436 raw_spin_lock_irqsave(&ifh->lock, flags); in perf_event_addr_filters_apply()
10437 list_for_each_entry(filter, &ifh->list, entry) { in perf_event_addr_filters_apply()
10438 if (filter->path.dentry) { in perf_event_addr_filters_apply()
10443 event->addr_filter_ranges[count].start = 0; in perf_event_addr_filters_apply()
10444 event->addr_filter_ranges[count].size = 0; in perf_event_addr_filters_apply()
10446 perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]); in perf_event_addr_filters_apply()
10448 event->addr_filter_ranges[count].start = filter->offset; in perf_event_addr_filters_apply()
10449 event->addr_filter_ranges[count].size = filter->size; in perf_event_addr_filters_apply()
10455 event->addr_filters_gen++; in perf_event_addr_filters_apply()
10456 raw_spin_unlock_irqrestore(&ifh->lock, flags); in perf_event_addr_filters_apply()
10458 if (ifh->nr_file_filters) { in perf_event_addr_filters_apply()
10488 IF_ACT_NONE = -1,
10527 int ret = -EINVAL; in perf_event_parse_addr_filter()
10531 return -ENOMEM; in perf_event_parse_addr_filter()
10539 ret = -EINVAL; in perf_event_parse_addr_filter()
10559 filter->action = actions[token]; in perf_event_parse_addr_filter()
10574 ret = kstrtoul(args[0].from, 0, &filter->offset); in perf_event_parse_addr_filter()
10580 ret = kstrtoul(args[1].from, 0, &filter->size); in perf_event_parse_addr_filter()
10591 ret = -ENOMEM; in perf_event_parse_addr_filter()
10609 ret = -EINVAL; in perf_event_parse_addr_filter()
10612 * ACTION "filter" must have a non-zero length region in perf_event_parse_addr_filter()
10615 if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER && in perf_event_parse_addr_filter()
10616 !filter->size) in perf_event_parse_addr_filter()
10624 * For now, we only support file-based filters in perf_event_parse_addr_filter()
10625 * in per-task events; doing so for CPU-wide in perf_event_parse_addr_filter()
10631 ret = -EOPNOTSUPP; in perf_event_parse_addr_filter()
10632 if (!event->ctx->task) in perf_event_parse_addr_filter()
10637 &filter->path); in perf_event_parse_addr_filter()
10641 ret = -EINVAL; in perf_event_parse_addr_filter()
10642 if (!filter->path.dentry || in perf_event_parse_addr_filter()
10643 !S_ISREG(d_inode(filter->path.dentry) in perf_event_parse_addr_filter()
10644 ->i_mode)) in perf_event_parse_addr_filter()
10647 event->addr_filters.nr_file_filters++; in perf_event_parse_addr_filter()
10685 lockdep_assert_held(&event->ctx->mutex); in perf_event_set_addr_filter()
10687 if (WARN_ON_ONCE(event->parent)) in perf_event_set_addr_filter()
10688 return -EINVAL; in perf_event_set_addr_filter()
10694 ret = event->pmu->addr_filters_validate(&filters); in perf_event_set_addr_filter()
10710 event->addr_filters.nr_file_filters = 0; in perf_event_set_addr_filter()
10717 int ret = -EINVAL; in perf_event_set_filter()
10726 struct perf_event_context *ctx = event->ctx; in perf_event_set_filter()
10731 * the tracepoint muck will deadlock against ctx->mutex, but in perf_event_set_filter()
10733 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we in perf_event_set_filter()
10739 mutex_unlock(&ctx->mutex); in perf_event_set_filter()
10740 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); in perf_event_set_filter()
10741 mutex_lock(&ctx->mutex); in perf_event_set_filter()
10765 if (event->state != PERF_EVENT_STATE_ACTIVE) in perf_swevent_hrtimer()
10768 event->pmu->read(event); in perf_swevent_hrtimer()
10770 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_swevent_hrtimer()
10774 if (!(event->attr.exclude_idle && is_idle_task(current))) in perf_swevent_hrtimer()
10779 period = max_t(u64, 10000, event->hw.sample_period); in perf_swevent_hrtimer()
10787 struct hw_perf_event *hwc = &event->hw; in perf_swevent_start_hrtimer()
10793 period = local64_read(&hwc->period_left); in perf_swevent_start_hrtimer()
10798 local64_set(&hwc->period_left, 0); in perf_swevent_start_hrtimer()
10800 period = max_t(u64, 10000, hwc->sample_period); in perf_swevent_start_hrtimer()
10802 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), in perf_swevent_start_hrtimer()
10808 struct hw_perf_event *hwc = &event->hw; in perf_swevent_cancel_hrtimer()
10811 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
10812 local64_set(&hwc->period_left, ktime_to_ns(remaining)); in perf_swevent_cancel_hrtimer()
10814 hrtimer_cancel(&hwc->hrtimer); in perf_swevent_cancel_hrtimer()
10820 struct hw_perf_event *hwc = &event->hw; in perf_swevent_init_hrtimer()
10825 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); in perf_swevent_init_hrtimer()
10826 hwc->hrtimer.function = perf_swevent_hrtimer; in perf_swevent_init_hrtimer()
10829 * Since hrtimers have a fixed rate, we can do a static freq->period in perf_swevent_init_hrtimer()
10832 if (event->attr.freq) { in perf_swevent_init_hrtimer()
10833 long freq = event->attr.sample_freq; in perf_swevent_init_hrtimer()
10835 event->attr.sample_period = NSEC_PER_SEC / freq; in perf_swevent_init_hrtimer()
10836 hwc->sample_period = event->attr.sample_period; in perf_swevent_init_hrtimer()
10837 local64_set(&hwc->period_left, hwc->sample_period); in perf_swevent_init_hrtimer()
10838 hwc->last_period = hwc->sample_period; in perf_swevent_init_hrtimer()
10839 event->attr.freq = 0; in perf_swevent_init_hrtimer()
10853 prev = local64_xchg(&event->hw.prev_count, now); in cpu_clock_event_update()
10854 local64_add(now - prev, &event->count); in cpu_clock_event_update()
10859 local64_set(&event->hw.prev_count, local_clock()); in cpu_clock_event_start()
10890 if (event->attr.type != PERF_TYPE_SOFTWARE) in cpu_clock_event_init()
10891 return -ENOENT; in cpu_clock_event_init()
10893 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) in cpu_clock_event_init()
10894 return -ENOENT; in cpu_clock_event_init()
10900 return -EOPNOTSUPP; in cpu_clock_event_init()
10929 prev = local64_xchg(&event->hw.prev_count, now); in task_clock_event_update()
10930 delta = now - prev; in task_clock_event_update()
10931 local64_add(delta, &event->count); in task_clock_event_update()
10936 local64_set(&event->hw.prev_count, event->ctx->time); in task_clock_event_start()
10943 task_clock_event_update(event, event->ctx->time); in task_clock_event_stop()
10963 u64 delta = now - event->ctx->timestamp; in task_clock_event_read()
10964 u64 time = event->ctx->time + delta; in task_clock_event_read()
10971 if (event->attr.type != PERF_TYPE_SOFTWARE) in task_clock_event_init()
10972 return -ENOENT; in task_clock_event_init()
10974 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) in task_clock_event_init()
10975 return -ENOENT; in task_clock_event_init()
10981 return -EOPNOTSUPP; in task_clock_event_init()
11073 if (pmu->task_ctx_nr == ctxn) in find_pmu_context()
11074 return pmu->pmu_cpu_context; in find_pmu_context()
11087 if (pmu->task_ctx_nr > perf_invalid_context) in free_pmu_context()
11090 free_percpu(pmu->pmu_cpu_context); in free_pmu_context()
11102 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); in nr_addr_filters_show()
11113 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->type); in type_show()
11124 return scnprintf(page, PAGE_SIZE - 1, "%d\n", pmu->hrtimer_interval_ms); in perf_event_mux_interval_ms_show()
11142 return -EINVAL; in perf_event_mux_interval_ms_store()
11145 if (timer == pmu->hrtimer_interval_ms) in perf_event_mux_interval_ms_store()
11149 pmu->hrtimer_interval_ms = timer; in perf_event_mux_interval_ms_store()
11155 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); in perf_event_mux_interval_ms_store()
11156 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); in perf_event_mux_interval_ms_store()
11188 int ret = -ENOMEM; in pmu_dev_alloc()
11190 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); in pmu_dev_alloc()
11191 if (!pmu->dev) in pmu_dev_alloc()
11194 pmu->dev->groups = pmu->attr_groups; in pmu_dev_alloc()
11195 device_initialize(pmu->dev); in pmu_dev_alloc()
11196 ret = dev_set_name(pmu->dev, "%s", pmu->name); in pmu_dev_alloc()
11200 dev_set_drvdata(pmu->dev, pmu); in pmu_dev_alloc()
11201 pmu->dev->bus = &pmu_bus; in pmu_dev_alloc()
11202 pmu->dev->release = pmu_dev_release; in pmu_dev_alloc()
11203 ret = device_add(pmu->dev); in pmu_dev_alloc()
11208 if (pmu->nr_addr_filters) in pmu_dev_alloc()
11209 ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters); in pmu_dev_alloc()
11214 if (pmu->attr_update) in pmu_dev_alloc()
11215 ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update); in pmu_dev_alloc()
11224 device_del(pmu->dev); in pmu_dev_alloc()
11227 put_device(pmu->dev); in pmu_dev_alloc()
11239 ret = -ENOMEM; in perf_pmu_register()
11240 pmu->pmu_disable_count = alloc_percpu(int); in perf_pmu_register()
11241 if (!pmu->pmu_disable_count) in perf_pmu_register()
11244 pmu->type = -1; in perf_pmu_register()
11247 pmu->name = name; in perf_pmu_register()
11261 pmu->type = type; in perf_pmu_register()
11270 if (pmu->task_ctx_nr == perf_hw_context) { in perf_pmu_register()
11279 !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS))) in perf_pmu_register()
11280 pmu->task_ctx_nr = perf_invalid_context; in perf_pmu_register()
11285 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); in perf_pmu_register()
11286 if (pmu->pmu_cpu_context) in perf_pmu_register()
11289 ret = -ENOMEM; in perf_pmu_register()
11290 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); in perf_pmu_register()
11291 if (!pmu->pmu_cpu_context) in perf_pmu_register()
11297 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); in perf_pmu_register()
11298 __perf_event_init_context(&cpuctx->ctx); in perf_pmu_register()
11299 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); in perf_pmu_register()
11300 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); in perf_pmu_register()
11301 cpuctx->ctx.pmu = pmu; in perf_pmu_register()
11302 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); in perf_pmu_register()
11306 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); in perf_pmu_register()
11307 cpuctx->heap = cpuctx->heap_default; in perf_pmu_register()
11311 if (!pmu->start_txn) { in perf_pmu_register()
11312 if (pmu->pmu_enable) { in perf_pmu_register()
11318 pmu->start_txn = perf_pmu_start_txn; in perf_pmu_register()
11319 pmu->commit_txn = perf_pmu_commit_txn; in perf_pmu_register()
11320 pmu->cancel_txn = perf_pmu_cancel_txn; in perf_pmu_register()
11322 pmu->start_txn = perf_pmu_nop_txn; in perf_pmu_register()
11323 pmu->commit_txn = perf_pmu_nop_int; in perf_pmu_register()
11324 pmu->cancel_txn = perf_pmu_nop_void; in perf_pmu_register()
11328 if (!pmu->pmu_enable) { in perf_pmu_register()
11329 pmu->pmu_enable = perf_pmu_nop_void; in perf_pmu_register()
11330 pmu->pmu_disable = perf_pmu_nop_void; in perf_pmu_register()
11333 if (!pmu->check_period) in perf_pmu_register()
11334 pmu->check_period = perf_event_nop_int; in perf_pmu_register()
11336 if (!pmu->event_idx) in perf_pmu_register()
11337 pmu->event_idx = perf_event_idx_default; in perf_pmu_register()
11345 list_add_rcu(&pmu->entry, &pmus); in perf_pmu_register()
11347 list_add_tail_rcu(&pmu->entry, &pmus); in perf_pmu_register()
11349 atomic_set(&pmu->exclusive_cnt, 0); in perf_pmu_register()
11357 device_del(pmu->dev); in perf_pmu_register()
11358 put_device(pmu->dev); in perf_pmu_register()
11361 if (pmu->type != PERF_TYPE_SOFTWARE) in perf_pmu_register()
11362 idr_remove(&pmu_idr, pmu->type); in perf_pmu_register()
11365 free_percpu(pmu->pmu_disable_count); in perf_pmu_register()
11373 list_del_rcu(&pmu->entry); in perf_pmu_unregister()
11382 free_percpu(pmu->pmu_disable_count); in perf_pmu_unregister()
11383 if (pmu->type != PERF_TYPE_SOFTWARE) in perf_pmu_unregister()
11384 idr_remove(&pmu_idr, pmu->type); in perf_pmu_unregister()
11386 if (pmu->nr_addr_filters) in perf_pmu_unregister()
11387 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); in perf_pmu_unregister()
11388 device_del(pmu->dev); in perf_pmu_unregister()
11389 put_device(pmu->dev); in perf_pmu_unregister()
11398 return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) || in has_extended_regs()
11399 (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK); in has_extended_regs()
11407 if (!try_module_get(pmu->module)) in perf_try_init_event()
11408 return -ENODEV; in perf_try_init_event()
11411 * A number of pmu->event_init() methods iterate the sibling_list to, in perf_try_init_event()
11413 * if this is a sibling event, acquire the ctx->mutex to protect in perf_try_init_event()
11416 if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) { in perf_try_init_event()
11418 * This ctx->mutex can nest when we're called through in perf_try_init_event()
11421 ctx = perf_event_ctx_lock_nested(event->group_leader, in perf_try_init_event()
11426 event->pmu = pmu; in perf_try_init_event()
11427 ret = pmu->event_init(event); in perf_try_init_event()
11430 perf_event_ctx_unlock(event->group_leader, ctx); in perf_try_init_event()
11433 if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && in perf_try_init_event()
11435 ret = -EOPNOTSUPP; in perf_try_init_event()
11437 if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE && in perf_try_init_event()
11439 ret = -EINVAL; in perf_try_init_event()
11441 if (ret && event->destroy) in perf_try_init_event()
11442 event->destroy(event); in perf_try_init_event()
11446 module_put(pmu->module); in perf_try_init_event()
11460 if (event->parent && event->parent->pmu) { in perf_init_event()
11461 pmu = event->parent->pmu; in perf_init_event()
11471 type = event->attr.type; in perf_init_event()
11473 type = event->attr.config >> PERF_PMU_TYPE_SHIFT; in perf_init_event()
11478 event->attr.config &= PERF_HW_EVENT_MASK; in perf_init_event()
11487 if (event->attr.type != type && type != PERF_TYPE_RAW && in perf_init_event()
11488 !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)) in perf_init_event()
11492 if (ret == -ENOENT && event->attr.type != type && !extended_type) { in perf_init_event()
11493 type = event->attr.type; in perf_init_event()
11508 if (ret != -ENOENT) { in perf_init_event()
11514 pmu = ERR_PTR(-ENOENT); in perf_init_event()
11523 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); in attach_sb_event()
11525 raw_spin_lock(&pel->lock); in attach_sb_event()
11526 list_add_rcu(&event->sb_list, &pel->list); in attach_sb_event()
11527 raw_spin_unlock(&pel->lock); in attach_sb_event()
11531 * We keep a list of all !task (and therefore per-cpu) events
11532 * that need to receive side-band records.
11534 * This avoids having to scan all the various PMU per-cpu contexts
11545 if (event->parent) in account_event_cpu()
11577 if (event->parent) in account_event()
11580 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) in account_event()
11582 if (event->attr.mmap || event->attr.mmap_data) in account_event()
11584 if (event->attr.build_id) in account_event()
11586 if (event->attr.comm) in account_event()
11588 if (event->attr.namespaces) in account_event()
11590 if (event->attr.cgroup) in account_event()
11592 if (event->attr.task) in account_event()
11594 if (event->attr.freq) in account_event()
11596 if (event->attr.context_switch) { in account_event()
11604 if (event->attr.ksymbol) in account_event()
11606 if (event->attr.bpf_event) in account_event()
11608 if (event->attr.text_poke) in account_event()
11632 * increments to by-pass the mutex. in account_event()
11639 account_event_cpu(event, event->cpu); in account_event()
11658 long err = -EINVAL; in perf_event_alloc()
11662 if (!task || cpu != -1) in perf_event_alloc()
11663 return ERR_PTR(-EINVAL); in perf_event_alloc()
11665 if (attr->sigtrap && !task) { in perf_event_alloc()
11667 return ERR_PTR(-EINVAL); in perf_event_alloc()
11670 node = (cpu >= 0) ? cpu_to_node(cpu) : -1; in perf_event_alloc()
11674 return ERR_PTR(-ENOMEM); in perf_event_alloc()
11683 mutex_init(&event->child_mutex); in perf_event_alloc()
11684 INIT_LIST_HEAD(&event->child_list); in perf_event_alloc()
11686 INIT_LIST_HEAD(&event->event_entry); in perf_event_alloc()
11687 INIT_LIST_HEAD(&event->sibling_list); in perf_event_alloc()
11688 INIT_LIST_HEAD(&event->active_list); in perf_event_alloc()
11690 INIT_LIST_HEAD(&event->rb_entry); in perf_event_alloc()
11691 INIT_LIST_HEAD(&event->active_entry); in perf_event_alloc()
11692 INIT_LIST_HEAD(&event->addr_filters.list); in perf_event_alloc()
11693 INIT_HLIST_NODE(&event->hlist_entry); in perf_event_alloc()
11696 init_waitqueue_head(&event->waitq); in perf_event_alloc()
11697 init_irq_work(&event->pending_irq, perf_pending_irq); in perf_event_alloc()
11698 init_task_work(&event->pending_task, perf_pending_task); in perf_event_alloc()
11700 mutex_init(&event->mmap_mutex); in perf_event_alloc()
11701 raw_spin_lock_init(&event->addr_filters.lock); in perf_event_alloc()
11703 atomic_long_set(&event->refcount, 1); in perf_event_alloc()
11704 event->cpu = cpu; in perf_event_alloc()
11705 event->attr = *attr; in perf_event_alloc()
11706 event->group_leader = group_leader; in perf_event_alloc()
11707 event->pmu = NULL; in perf_event_alloc()
11708 event->oncpu = -1; in perf_event_alloc()
11710 event->parent = parent_event; in perf_event_alloc()
11712 event->ns = get_pid_ns(task_active_pid_ns(current)); in perf_event_alloc()
11713 event->id = atomic64_inc_return(&perf_event_id); in perf_event_alloc()
11715 event->state = PERF_EVENT_STATE_INACTIVE; in perf_event_alloc()
11718 event->event_caps = parent_event->event_caps; in perf_event_alloc()
11721 event->attach_state = PERF_ATTACH_TASK; in perf_event_alloc()
11727 event->hw.target = get_task_struct(task); in perf_event_alloc()
11730 event->clock = &local_clock; in perf_event_alloc()
11732 event->clock = parent_event->clock; in perf_event_alloc()
11735 overflow_handler = parent_event->overflow_handler; in perf_event_alloc()
11736 context = parent_event->overflow_handler_context; in perf_event_alloc()
11739 struct bpf_prog *prog = parent_event->prog; in perf_event_alloc()
11742 event->prog = prog; in perf_event_alloc()
11743 event->orig_overflow_handler = in perf_event_alloc()
11744 parent_event->orig_overflow_handler; in perf_event_alloc()
11750 event->overflow_handler = overflow_handler; in perf_event_alloc()
11751 event->overflow_handler_context = context; in perf_event_alloc()
11753 event->overflow_handler = perf_event_output_backward; in perf_event_alloc()
11754 event->overflow_handler_context = NULL; in perf_event_alloc()
11756 event->overflow_handler = perf_event_output_forward; in perf_event_alloc()
11757 event->overflow_handler_context = NULL; in perf_event_alloc()
11764 hwc = &event->hw; in perf_event_alloc()
11765 hwc->sample_period = attr->sample_period; in perf_event_alloc()
11766 if (attr->freq && attr->sample_freq) in perf_event_alloc()
11767 hwc->sample_period = 1; in perf_event_alloc()
11768 hwc->last_period = hwc->sample_period; in perf_event_alloc()
11770 local64_set(&hwc->period_left, hwc->sample_period); in perf_event_alloc()
11776 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_READ)) in perf_event_alloc()
11780 event->attr.branch_sample_type = 0; in perf_event_alloc()
11789 * Disallow uncore-cgroup events, they don't make sense as the cgroup will in perf_event_alloc()
11792 if (pmu->task_ctx_nr == perf_invalid_context && cgroup_fd != -1) { in perf_event_alloc()
11793 err = -EINVAL; in perf_event_alloc()
11797 if (event->attr.aux_output && in perf_event_alloc()
11798 !(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT)) { in perf_event_alloc()
11799 err = -EOPNOTSUPP; in perf_event_alloc()
11803 if (cgroup_fd != -1) { in perf_event_alloc()
11814 event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters, in perf_event_alloc()
11817 if (!event->addr_filter_ranges) { in perf_event_alloc()
11818 err = -ENOMEM; in perf_event_alloc()
11826 if (event->parent) { in perf_event_alloc()
11829 raw_spin_lock_irq(&ifh->lock); in perf_event_alloc()
11830 memcpy(event->addr_filter_ranges, in perf_event_alloc()
11831 event->parent->addr_filter_ranges, in perf_event_alloc()
11832 pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range)); in perf_event_alloc()
11833 raw_spin_unlock_irq(&ifh->lock); in perf_event_alloc()
11837 event->addr_filters_gen = 1; in perf_event_alloc()
11840 if (!event->parent) { in perf_event_alloc()
11841 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { in perf_event_alloc()
11842 err = get_callchain_buffers(attr->sample_max_stack); in perf_event_alloc()
11858 if (!event->parent) { in perf_event_alloc()
11859 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) in perf_event_alloc()
11863 kfree(event->addr_filter_ranges); in perf_event_alloc()
11871 if (event->destroy) in perf_event_alloc()
11872 event->destroy(event); in perf_event_alloc()
11873 module_put(pmu->module); in perf_event_alloc()
11875 if (event->hw.target) in perf_event_alloc()
11876 put_task_struct(event->hw.target); in perf_event_alloc()
11877 call_rcu(&event->rcu_head, free_event_rcu); in perf_event_alloc()
11891 ret = get_user(size, &uattr->size); in perf_copy_attr()
11903 if (ret == -E2BIG) in perf_copy_attr()
11908 attr->size = size; in perf_copy_attr()
11910 if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) in perf_copy_attr()
11911 return -EINVAL; in perf_copy_attr()
11913 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) in perf_copy_attr()
11914 return -EINVAL; in perf_copy_attr()
11916 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) in perf_copy_attr()
11917 return -EINVAL; in perf_copy_attr()
11919 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { in perf_copy_attr()
11920 u64 mask = attr->branch_sample_type; in perf_copy_attr()
11923 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) in perf_copy_attr()
11924 return -EINVAL; in perf_copy_attr()
11928 return -EINVAL; in perf_copy_attr()
11934 if (!attr->exclude_kernel) in perf_copy_attr()
11937 if (!attr->exclude_user) in perf_copy_attr()
11940 if (!attr->exclude_hv) in perf_copy_attr()
11945 attr->branch_sample_type = mask; in perf_copy_attr()
11955 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { in perf_copy_attr()
11956 ret = perf_reg_validate(attr->sample_regs_user); in perf_copy_attr()
11961 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { in perf_copy_attr()
11963 return -ENOSYS; in perf_copy_attr()
11970 if (attr->sample_stack_user >= USHRT_MAX) in perf_copy_attr()
11971 return -EINVAL; in perf_copy_attr()
11972 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) in perf_copy_attr()
11973 return -EINVAL; in perf_copy_attr()
11976 if (!attr->sample_max_stack) in perf_copy_attr()
11977 attr->sample_max_stack = sysctl_perf_event_max_stack; in perf_copy_attr()
11979 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) in perf_copy_attr()
11980 ret = perf_reg_validate(attr->sample_regs_intr); in perf_copy_attr()
11983 if (attr->sample_type & PERF_SAMPLE_CGROUP) in perf_copy_attr()
11984 return -EINVAL; in perf_copy_attr()
11986 if ((attr->sample_type & PERF_SAMPLE_WEIGHT) && in perf_copy_attr()
11987 (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) in perf_copy_attr()
11988 return -EINVAL; in perf_copy_attr()
11990 if (!attr->inherit && attr->inherit_thread) in perf_copy_attr()
11991 return -EINVAL; in perf_copy_attr()
11993 if (attr->remove_on_exec && attr->enable_on_exec) in perf_copy_attr()
11994 return -EINVAL; in perf_copy_attr()
11996 if (attr->sigtrap && !attr->remove_on_exec) in perf_copy_attr()
11997 return -EINVAL; in perf_copy_attr()
12003 put_user(sizeof(*attr), &uattr->size); in perf_copy_attr()
12004 ret = -E2BIG; in perf_copy_attr()
12021 int ret = -EINVAL; in perf_event_set_output()
12024 mutex_lock(&event->mmap_mutex); in perf_event_set_output()
12033 * Don't allow cross-cpu buffers in perf_event_set_output()
12035 if (output_event->cpu != event->cpu) in perf_event_set_output()
12039 * If its not a per-cpu rb, it must be the same task. in perf_event_set_output()
12041 if (output_event->cpu == -1 && output_event->ctx != event->ctx) in perf_event_set_output()
12047 if (output_event->clock != event->clock) in perf_event_set_output()
12061 event->pmu != output_event->pmu) in perf_event_set_output()
12066 * output_event is already on rb->event_list, and the list iteration in perf_event_set_output()
12069 * observe !rb->mmap_count. in perf_event_set_output()
12071 mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex); in perf_event_set_output()
12074 if (atomic_read(&event->mmap_count)) in perf_event_set_output()
12084 if (!atomic_read(&rb->mmap_count)) { in perf_event_set_output()
12094 mutex_unlock(&event->mmap_mutex); in perf_event_set_output()
12096 mutex_unlock(&output_event->mmap_mutex); in perf_event_set_output()
12108 event->clock = &ktime_get_mono_fast_ns; in perf_event_set_clock()
12113 event->clock = &ktime_get_raw_fast_ns; in perf_event_set_clock()
12118 event->clock = &ktime_get_real_ns; in perf_event_set_clock()
12122 event->clock = &ktime_get_boottime_ns; in perf_event_set_clock()
12126 event->clock = &ktime_get_clocktai_ns; in perf_event_set_clock()
12130 return -EINVAL; in perf_event_set_clock()
12133 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) in perf_event_set_clock()
12134 return -EINVAL; in perf_event_set_clock()
12151 gctx = READ_ONCE(group_leader->ctx); in __perf_event_ctx_lock_double()
12152 if (!refcount_inc_not_zero(&gctx->refcount)) { in __perf_event_ctx_lock_double()
12158 mutex_lock_double(&gctx->mutex, &ctx->mutex); in __perf_event_ctx_lock_double()
12160 if (group_leader->ctx != gctx) { in __perf_event_ctx_lock_double()
12161 mutex_unlock(&ctx->mutex); in __perf_event_ctx_lock_double()
12162 mutex_unlock(&gctx->mutex); in __perf_event_ctx_lock_double()
12176 if (attr->sigtrap) { in perf_check_permission()
12182 is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL); in perf_check_permission()
12202 * sys_perf_event_open - open a performance event, associate it to a task/cpu
12226 int cgroup_fd = -1; in SYSCALL_DEFINE5()
12230 return -EINVAL; in SYSCALL_DEFINE5()
12249 return -EACCES; in SYSCALL_DEFINE5()
12254 return -EINVAL; in SYSCALL_DEFINE5()
12257 return -EINVAL; in SYSCALL_DEFINE5()
12280 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) in SYSCALL_DEFINE5()
12281 return -EINVAL; in SYSCALL_DEFINE5()
12290 if (group_fd != -1) { in SYSCALL_DEFINE5()
12294 group_leader = group.file->private_data; in SYSCALL_DEFINE5()
12301 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { in SYSCALL_DEFINE5()
12310 group_leader->attr.inherit != attr.inherit) { in SYSCALL_DEFINE5()
12311 err = -EINVAL; in SYSCALL_DEFINE5()
12326 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { in SYSCALL_DEFINE5()
12327 err = -EOPNOTSUPP; in SYSCALL_DEFINE5()
12336 pmu = event->pmu; in SYSCALL_DEFINE5()
12344 if (pmu->task_ctx_nr == perf_sw_context) in SYSCALL_DEFINE5()
12345 event->event_caps |= PERF_EV_CAP_SOFTWARE; in SYSCALL_DEFINE5()
12358 pmu = group_leader->ctx->pmu; in SYSCALL_DEFINE5()
12361 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { in SYSCALL_DEFINE5()
12384 err = -EINVAL; in SYSCALL_DEFINE5()
12388 * becoming part of another group-sibling): in SYSCALL_DEFINE5()
12390 if (group_leader->group_leader != group_leader) in SYSCALL_DEFINE5()
12394 if (group_leader->clock != event->clock) in SYSCALL_DEFINE5()
12402 if (group_leader->cpu != event->cpu) in SYSCALL_DEFINE5()
12407 * per-CPU events. in SYSCALL_DEFINE5()
12409 if (group_leader->ctx->task != ctx->task) in SYSCALL_DEFINE5()
12417 * Racy, not holding group_leader->ctx->mutex, see comment with in SYSCALL_DEFINE5()
12420 if (!move_group && group_leader->ctx != ctx) in SYSCALL_DEFINE5()
12445 err = down_read_interruptible(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12455 err = -EACCES; in SYSCALL_DEFINE5()
12463 if (gctx->task == TASK_TOMBSTONE) { in SYSCALL_DEFINE5()
12464 err = -ESRCH; in SYSCALL_DEFINE5()
12472 if (!(group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { in SYSCALL_DEFINE5()
12479 err = -EINVAL; in SYSCALL_DEFINE5()
12489 * Failure to create exclusive events returns -EBUSY. in SYSCALL_DEFINE5()
12491 err = -EBUSY; in SYSCALL_DEFINE5()
12500 mutex_lock(&ctx->mutex); in SYSCALL_DEFINE5()
12503 * Now that we hold ctx->lock, (re)validate group_leader->ctx == ctx, in SYSCALL_DEFINE5()
12506 if (group_leader && group_leader->ctx != ctx) { in SYSCALL_DEFINE5()
12507 err = -EINVAL; in SYSCALL_DEFINE5()
12513 if (ctx->task == TASK_TOMBSTONE) { in SYSCALL_DEFINE5()
12514 err = -ESRCH; in SYSCALL_DEFINE5()
12519 err = -E2BIG; in SYSCALL_DEFINE5()
12533 if (!cpuctx->online) { in SYSCALL_DEFINE5()
12534 err = -ENODEV; in SYSCALL_DEFINE5()
12540 err = -EINVAL; in SYSCALL_DEFINE5()
12549 err = -EBUSY; in SYSCALL_DEFINE5()
12553 WARN_ON_ONCE(ctx->parent_ctx); in SYSCALL_DEFINE5()
12583 * (through the sibling list, which is still in-tact), we can in SYSCALL_DEFINE5()
12586 * By installing siblings first we NO-OP because they're not in SYSCALL_DEFINE5()
12591 perf_install_in_context(ctx, sibling, sibling->cpu); in SYSCALL_DEFINE5()
12601 perf_install_in_context(ctx, group_leader, group_leader->cpu); in SYSCALL_DEFINE5()
12614 event->owner = current; in SYSCALL_DEFINE5()
12616 perf_install_in_context(ctx, event, event->cpu); in SYSCALL_DEFINE5()
12621 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
12624 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12628 mutex_lock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
12629 list_add_tail(&event->owner_entry, ¤t->perf_event_list); in SYSCALL_DEFINE5()
12630 mutex_unlock(¤t->perf_event_mutex); in SYSCALL_DEFINE5()
12645 mutex_unlock(&ctx->mutex); in SYSCALL_DEFINE5()
12648 up_read(&task->signal->exec_update_lock); in SYSCALL_DEFINE5()
12656 * If event_file is set, the fput() above will have called ->release() in SYSCALL_DEFINE5()
12694 if (attr->aux_output) in perf_event_create_kernel_counter()
12695 return ERR_PTR(-EINVAL); in perf_event_create_kernel_counter()
12698 overflow_handler, context, -1); in perf_event_create_kernel_counter()
12705 event->owner = TASK_TOMBSTONE; in perf_event_create_kernel_counter()
12710 ctx = find_get_context(event->pmu, task, event); in perf_event_create_kernel_counter()
12716 WARN_ON_ONCE(ctx->parent_ctx); in perf_event_create_kernel_counter()
12717 mutex_lock(&ctx->mutex); in perf_event_create_kernel_counter()
12718 if (ctx->task == TASK_TOMBSTONE) { in perf_event_create_kernel_counter()
12719 err = -ESRCH; in perf_event_create_kernel_counter()
12732 if (!cpuctx->online) { in perf_event_create_kernel_counter()
12733 err = -ENODEV; in perf_event_create_kernel_counter()
12739 err = -EBUSY; in perf_event_create_kernel_counter()
12743 perf_install_in_context(ctx, event, event->cpu); in perf_event_create_kernel_counter()
12745 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
12750 mutex_unlock(&ctx->mutex); in perf_event_create_kernel_counter()
12767 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; in perf_pmu_migrate_context()
12768 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; in perf_pmu_migrate_context()
12774 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); in perf_pmu_migrate_context()
12775 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, in perf_pmu_migrate_context()
12780 list_add(&event->migrate_entry, &events); in perf_pmu_migrate_context()
12784 * Wait for the events to quiesce before re-instating them. in perf_pmu_migrate_context()
12789 * Re-instate events in 2 passes. in perf_pmu_migrate_context()
12797 if (event->group_leader == event) in perf_pmu_migrate_context()
12800 list_del(&event->migrate_entry); in perf_pmu_migrate_context()
12801 if (event->state >= PERF_EVENT_STATE_OFF) in perf_pmu_migrate_context()
12802 event->state = PERF_EVENT_STATE_INACTIVE; in perf_pmu_migrate_context()
12813 list_del(&event->migrate_entry); in perf_pmu_migrate_context()
12814 if (event->state >= PERF_EVENT_STATE_OFF) in perf_pmu_migrate_context()
12815 event->state = PERF_EVENT_STATE_INACTIVE; in perf_pmu_migrate_context()
12820 mutex_unlock(&dst_ctx->mutex); in perf_pmu_migrate_context()
12821 mutex_unlock(&src_ctx->mutex); in perf_pmu_migrate_context()
12827 struct perf_event *parent_event = child_event->parent; in sync_child_event()
12830 if (child_event->attr.inherit_stat) { in sync_child_event()
12831 struct task_struct *task = child_event->ctx->task; in sync_child_event()
12842 atomic64_add(child_val, &parent_event->child_count); in sync_child_event()
12843 atomic64_add(child_event->total_time_enabled, in sync_child_event()
12844 &parent_event->child_total_time_enabled); in sync_child_event()
12845 atomic64_add(child_event->total_time_running, in sync_child_event()
12846 &parent_event->child_total_time_running); in sync_child_event()
12852 struct perf_event *parent_event = event->parent; in perf_event_exit_event()
12869 mutex_lock(&parent_event->child_mutex); in perf_event_exit_event()
12874 raw_spin_lock_irq(&ctx->lock); in perf_event_exit_event()
12875 if (event->state > PERF_EVENT_STATE_EXIT) in perf_event_exit_event()
12877 raw_spin_unlock_irq(&ctx->lock); in perf_event_exit_event()
12883 mutex_unlock(&parent_event->child_mutex); in perf_event_exit_event()
12911 * In order to reduce the amount of tricky in ctx tear-down, we hold in perf_event_exit_task_context()
12920 mutex_lock(&child_ctx->mutex); in perf_event_exit_task_context()
12923 * In a single ctx::lock section, de-schedule the events and detach the in perf_event_exit_task_context()
12927 raw_spin_lock_irq(&child_ctx->lock); in perf_event_exit_task_context()
12931 * Now that the context is inactive, destroy the task <-> ctx relation in perf_event_exit_task_context()
12934 RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL); in perf_event_exit_task_context()
12936 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); in perf_event_exit_task_context()
12940 raw_spin_unlock_irq(&child_ctx->lock); in perf_event_exit_task_context()
12952 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) in perf_event_exit_task_context()
12955 mutex_unlock(&child_ctx->mutex); in perf_event_exit_task_context()
12971 mutex_lock(&child->perf_event_mutex); in perf_event_exit_task()
12972 list_for_each_entry_safe(event, tmp, &child->perf_event_list, in perf_event_exit_task()
12974 list_del_init(&event->owner_entry); in perf_event_exit_task()
12979 * we need to serialize on the owner->perf_event_mutex. in perf_event_exit_task()
12981 smp_store_release(&event->owner, NULL); in perf_event_exit_task()
12983 mutex_unlock(&child->perf_event_mutex); in perf_event_exit_task()
12991 * child contexts and sets child->perf_event_ctxp[] to NULL. in perf_event_exit_task()
13000 struct perf_event *parent = event->parent; in perf_free_event()
13005 mutex_lock(&parent->child_mutex); in perf_free_event()
13006 list_del_init(&event->child_list); in perf_free_event()
13007 mutex_unlock(&parent->child_mutex); in perf_free_event()
13011 raw_spin_lock_irq(&ctx->lock); in perf_free_event()
13014 raw_spin_unlock_irq(&ctx->lock); in perf_free_event()
13032 ctx = task->perf_event_ctxp[ctxn]; in perf_event_free_task()
13036 mutex_lock(&ctx->mutex); in perf_event_free_task()
13037 raw_spin_lock_irq(&ctx->lock); in perf_event_free_task()
13039 * Destroy the task <-> ctx relation and mark the context dead. in perf_event_free_task()
13044 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], NULL); in perf_event_free_task()
13045 WRITE_ONCE(ctx->task, TASK_TOMBSTONE); in perf_event_free_task()
13047 raw_spin_unlock_irq(&ctx->lock); in perf_event_free_task()
13049 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) in perf_event_free_task()
13052 mutex_unlock(&ctx->mutex); in perf_event_free_task()
13063 * _free_event()'s put_task_struct(event->hw.target) will be a in perf_event_free_task()
13064 * use-after-free. in perf_event_free_task()
13068 wait_var_event(&ctx->refcount, refcount_read(&ctx->refcount) == 1); in perf_event_free_task()
13078 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); in perf_event_delayed_put()
13085 return ERR_PTR(-EBADF); in perf_event_get()
13087 if (file->f_op != &perf_fops) { in perf_event_get()
13089 return ERR_PTR(-EBADF); in perf_event_get()
13097 if (file->f_op != &perf_fops) in perf_get_event()
13098 return ERR_PTR(-EINVAL); in perf_get_event()
13100 return file->private_data; in perf_get_event()
13106 return ERR_PTR(-EINVAL); in perf_event_attrs()
13108 return &event->attr; in perf_event_attrs()
13115 * - valid pointer on success
13116 * - NULL for orphaned events
13117 * - IS_ERR() on error
13127 enum perf_event_state parent_state = parent_event->state; in inherit_event()
13137 if (parent_event->parent) in inherit_event()
13138 parent_event = parent_event->parent; in inherit_event()
13140 child_event = perf_event_alloc(&parent_event->attr, in inherit_event()
13141 parent_event->cpu, in inherit_event()
13144 NULL, NULL, -1); in inherit_event()
13149 if ((child_event->attach_state & PERF_ATTACH_TASK_DATA) && in inherit_event()
13150 !child_ctx->task_ctx_data) { in inherit_event()
13151 struct pmu *pmu = child_event->pmu; in inherit_event()
13153 child_ctx->task_ctx_data = alloc_task_ctx_data(pmu); in inherit_event()
13154 if (!child_ctx->task_ctx_data) { in inherit_event()
13156 return ERR_PTR(-ENOMEM); in inherit_event()
13161 * is_orphaned_event() and list_add_tail(&parent_event->child_list) in inherit_event()
13166 mutex_lock(&parent_event->child_mutex); in inherit_event()
13168 !atomic_long_inc_not_zero(&parent_event->refcount)) { in inherit_event()
13169 mutex_unlock(&parent_event->child_mutex); in inherit_event()
13183 child_event->state = PERF_EVENT_STATE_INACTIVE; in inherit_event()
13185 child_event->state = PERF_EVENT_STATE_OFF; in inherit_event()
13187 if (parent_event->attr.freq) { in inherit_event()
13188 u64 sample_period = parent_event->hw.sample_period; in inherit_event()
13189 struct hw_perf_event *hwc = &child_event->hw; in inherit_event()
13191 hwc->sample_period = sample_period; in inherit_event()
13192 hwc->last_period = sample_period; in inherit_event()
13194 local64_set(&hwc->period_left, sample_period); in inherit_event()
13197 child_event->ctx = child_ctx; in inherit_event()
13198 child_event->overflow_handler = parent_event->overflow_handler; in inherit_event()
13199 child_event->overflow_handler_context in inherit_event()
13200 = parent_event->overflow_handler_context; in inherit_event()
13211 raw_spin_lock_irqsave(&child_ctx->lock, flags); in inherit_event()
13213 child_event->attach_state |= PERF_ATTACH_CHILD; in inherit_event()
13214 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); in inherit_event()
13219 list_add_tail(&child_event->child_list, &parent_event->child_list); in inherit_event()
13220 mutex_unlock(&parent_event->child_mutex); in inherit_event()
13232 * - 0 on success
13233 * - <0 on error
13260 if (sub->aux_event == parent_event && child_ctr && in inherit_group()
13262 return -EINVAL; in inherit_group()
13268 * Creates the child task context and tries to inherit the event-group.
13275 * - 0 on success
13276 * - <0 on error
13287 if (!event->attr.inherit || in inherit_task_group()
13288 (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) || in inherit_task_group()
13290 (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) { in inherit_task_group()
13295 child_ctx = child->perf_event_ctxp[ctxn]; in inherit_task_group()
13303 child_ctx = alloc_perf_context(parent_ctx->pmu, child); in inherit_task_group()
13305 return -ENOMEM; in inherit_task_group()
13307 child->perf_event_ctxp[ctxn] = child_ctx; in inherit_task_group()
13333 if (likely(!parent->perf_event_ctxp[ctxn])) in perf_event_init_context()
13346 * it non-NULL earlier, the only reason for it to become NULL in perf_event_init_context()
13352 * Lock the parent list. No need to lock the child - not PID in perf_event_init_context()
13355 mutex_lock(&parent_ctx->mutex); in perf_event_init_context()
13358 * We dont have to disable NMIs - we are only looking at in perf_event_init_context()
13361 perf_event_groups_for_each(event, &parent_ctx->pinned_groups) { in perf_event_init_context()
13370 * We can't hold ctx->lock when iterating the ->flexible_group list due in perf_event_init_context()
13374 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
13375 parent_ctx->rotate_disable = 1; in perf_event_init_context()
13376 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
13378 perf_event_groups_for_each(event, &parent_ctx->flexible_groups) { in perf_event_init_context()
13386 raw_spin_lock_irqsave(&parent_ctx->lock, flags); in perf_event_init_context()
13387 parent_ctx->rotate_disable = 0; in perf_event_init_context()
13389 child_ctx = child->perf_event_ctxp[ctxn]; in perf_event_init_context()
13397 * parent_ctx->lock avoids it from being uncloned. in perf_event_init_context()
13399 cloned_ctx = parent_ctx->parent_ctx; in perf_event_init_context()
13401 child_ctx->parent_ctx = cloned_ctx; in perf_event_init_context()
13402 child_ctx->parent_gen = parent_ctx->parent_gen; in perf_event_init_context()
13404 child_ctx->parent_ctx = parent_ctx; in perf_event_init_context()
13405 child_ctx->parent_gen = parent_ctx->generation; in perf_event_init_context()
13407 get_ctx(child_ctx->parent_ctx); in perf_event_init_context()
13410 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); in perf_event_init_context()
13412 mutex_unlock(&parent_ctx->mutex); in perf_event_init_context()
13427 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); in perf_event_init_task()
13428 mutex_init(&child->perf_event_mutex); in perf_event_init_task()
13429 INIT_LIST_HEAD(&child->perf_event_list); in perf_event_init_task()
13451 mutex_init(&swhash->hlist_mutex); in perf_event_init_all_cpus()
13468 mutex_lock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
13469 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { in perf_swevent_init_cpu()
13474 rcu_assign_pointer(swhash->swevent_hlist, hlist); in perf_swevent_init_cpu()
13476 mutex_unlock(&swhash->hlist_mutex); in perf_swevent_init_cpu()
13486 raw_spin_lock(&ctx->lock); in __perf_event_exit_context()
13488 list_for_each_entry(event, &ctx->event_list, event_entry) in __perf_event_exit_context()
13490 raw_spin_unlock(&ctx->lock); in __perf_event_exit_context()
13501 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); in perf_event_exit_cpu_context()
13502 ctx = &cpuctx->ctx; in perf_event_exit_cpu_context()
13504 mutex_lock(&ctx->mutex); in perf_event_exit_cpu_context()
13506 cpuctx->online = 0; in perf_event_exit_cpu_context()
13507 mutex_unlock(&ctx->mutex); in perf_event_exit_cpu_context()
13529 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); in perf_event_init_cpu()
13530 ctx = &cpuctx->ctx; in perf_event_init_cpu()
13532 mutex_lock(&ctx->mutex); in perf_event_init_cpu()
13533 cpuctx->online = 1; in perf_event_init_cpu()
13534 mutex_unlock(&ctx->mutex); in perf_event_init_cpu()
13576 perf_pmu_register(&perf_cpu_clock, NULL, -1); in perf_event_init()
13577 perf_pmu_register(&perf_task_clock, NULL, -1); in perf_event_init()
13601 if (pmu_attr->event_str) in perf_event_sysfs_show()
13602 return sprintf(page, "%s\n", pmu_attr->event_str); in perf_event_sysfs_show()
13620 if (!pmu->name || pmu->type < 0) in perf_event_sysfs_init()
13624 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); in perf_event_sysfs_init()
13644 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
13646 jc->info = alloc_percpu(struct perf_cgroup_info); in perf_cgroup_css_alloc()
13647 if (!jc->info) { in perf_cgroup_css_alloc()
13649 return ERR_PTR(-ENOMEM); in perf_cgroup_css_alloc()
13652 return &jc->css; in perf_cgroup_css_alloc()
13659 free_percpu(jc->info); in perf_cgroup_css_free()
13665 perf_event_cgroup(css->cgroup); in perf_cgroup_css_online()