170 static bool is_kernel_event(struct perf_event *event)  in is_kernel_event()  argument
172 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;  in is_kernel_event()
198 	struct perf_event *event;  member
206 	struct perf_event *event = efs->event;  in event_function()  local
207 	struct perf_event_context *ctx = event->ctx;  in event_function()
242 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
249 static void event_function_call(struct perf_event *event, event_f func, void *data)  in event_function_call()  argument
251 	struct perf_event_context *ctx = event->ctx;  in event_function_call()
254 		.event = event,  in event_function_call()
259 	if (!event->parent) {  in event_function_call()
269 		cpu_function_call(event->cpu, event_function, &efs);  in event_function_call()
294 	func(event, NULL, ctx, data);  in event_function_call()
302 static void event_function_local(struct perf_event *event, event_f func, void *data)  in event_function_local()  argument
304 	struct perf_event_context *ctx = event->ctx;  in event_function_local()
341 	func(event, cpuctx, ctx, data);  in event_function_local()
566 static u64 perf_event_time(struct perf_event *event);
580 static inline u64 perf_event_clock(struct perf_event *event)  in perf_event_clock()  argument
582 	return event->clock();  in perf_event_clock()
608 __perf_effective_state(struct perf_event *event)  in __perf_effective_state()  argument
610 	struct perf_event *leader = event->group_leader;  in __perf_effective_state()
615 	return event->state;  in __perf_effective_state()
619 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)  in __perf_update_times()  argument
621 	enum perf_event_state state = __perf_effective_state(event);  in __perf_update_times()
622 	u64 delta = now - event->tstamp;  in __perf_update_times()
624 	*enabled = event->total_time_enabled;  in __perf_update_times()
628 	*running = event->total_time_running;  in __perf_update_times()
633 static void perf_event_update_time(struct perf_event *event)  in perf_event_update_time()  argument
635 	u64 now = perf_event_time(event);  in perf_event_update_time()
637 	__perf_update_times(event, now, &event->total_time_enabled,  in perf_event_update_time()
638 					&event->total_time_running);  in perf_event_update_time()
639 	event->tstamp = now;  in perf_event_update_time()
651 perf_event_set_state(struct perf_event *event, enum perf_event_state state)  in perf_event_set_state()  argument
653 	if (event->state == state)  in perf_event_set_state()
656 	perf_event_update_time(event);  in perf_event_set_state()
661 	if ((event->state < 0) ^ (state < 0))  in perf_event_set_state()
662 		perf_event_update_sibling_time(event);  in perf_event_set_state()
664 	WRITE_ONCE(event->state, state);  in perf_event_set_state()
670 perf_cgroup_match(struct perf_event *event)  in perf_cgroup_match()  argument
672 	struct perf_event_context *ctx = event->ctx;  in perf_cgroup_match()
676 	if (!event->cgrp)  in perf_cgroup_match()
690 				    event->cgrp->css.cgroup);  in perf_cgroup_match()
693 static inline void perf_detach_cgroup(struct perf_event *event)  in perf_detach_cgroup()  argument
695 	css_put(&event->cgrp->css);  in perf_detach_cgroup()
696 	event->cgrp = NULL;  in perf_detach_cgroup()
699 static inline int is_cgroup_event(struct perf_event *event)  in is_cgroup_event()  argument
701 	return event->cgrp != NULL;  in is_cgroup_event()
704 static inline u64 perf_cgroup_event_time(struct perf_event *event)  in perf_cgroup_event_time()  argument
708 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time()
738 static inline void update_cgrp_time_from_event(struct perf_event *event)  in update_cgrp_time_from_event()  argument
746 	if (!is_cgroup_event(event))  in update_cgrp_time_from_event()
749 	cgrp = perf_cgroup_from_task(current, event->ctx);  in update_cgrp_time_from_event()
753        if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))  in update_cgrp_time_from_event()
754 		__update_cgrp_time(event->cgrp);  in update_cgrp_time_from_event()
893 static inline int perf_cgroup_connect(int fd, struct perf_event *event,  in perf_cgroup_connect()  argument
913 	event->cgrp = cgrp;  in perf_cgroup_connect()
921 		perf_detach_cgroup(event);  in perf_cgroup_connect()
930 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)  in perf_cgroup_set_shadow_time()  argument
933 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_set_shadow_time()
934 	event->shadow_ctx_time = now - t->timestamp;  in perf_cgroup_set_shadow_time()
942 list_update_cgroup_event(struct perf_event *event,  in list_update_cgroup_event()  argument
948 	if (!is_cgroup_event(event))  in list_update_cgroup_event()
966 		if (cgroup_is_descendant(cgrp->css.cgroup, event->cgrp->css.cgroup))  in list_update_cgroup_event()
989 perf_cgroup_match(struct perf_event *event)  in perf_cgroup_match()  argument
994 static inline void perf_detach_cgroup(struct perf_event *event)  in perf_detach_cgroup()  argument
997 static inline int is_cgroup_event(struct perf_event *event)  in is_cgroup_event()  argument
1002 static inline void update_cgrp_time_from_event(struct perf_event *event)  in update_cgrp_time_from_event()  argument
1020 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,  in perf_cgroup_connect()  argument
1039 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)  in perf_cgroup_set_shadow_time()  argument
1043 static inline u64 perf_cgroup_event_time(struct perf_event *event)  in perf_cgroup_event_time()  argument
1049 list_update_cgroup_event(struct perf_event *event,  in list_update_cgroup_event()  argument
1263 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)  in perf_event_ctx_lock_nested()  argument
1269 	ctx = READ_ONCE(event->ctx);  in perf_event_ctx_lock_nested()
1277 	if (event->ctx != ctx) {  in perf_event_ctx_lock_nested()
1287 perf_event_ctx_lock(struct perf_event *event)  in perf_event_ctx_lock()  argument
1289 	return perf_event_ctx_lock_nested(event, 0);  in perf_event_ctx_lock()
1292 static void perf_event_ctx_unlock(struct perf_event *event,  in perf_event_ctx_unlock()  argument
1318 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,  in perf_event_pid_type()  argument
1325 	if (event->parent)  in perf_event_pid_type()
1326 		event = event->parent;  in perf_event_pid_type()
1328 	nr = __task_pid_nr_ns(p, type, event->ns);  in perf_event_pid_type()
1335 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)  in perf_event_pid()  argument
1337 	return perf_event_pid_type(event, p, PIDTYPE_TGID);  in perf_event_pid()
1340 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)  in perf_event_tid()  argument
1342 	return perf_event_pid_type(event, p, PIDTYPE_PID);  in perf_event_tid()
1349 static u64 primary_event_id(struct perf_event *event)  in primary_event_id()  argument
1351 	u64 id = event->id;  in primary_event_id()
1353 	if (event->parent)  in primary_event_id()
1354 		id = event->parent->id;  in primary_event_id()
1455 static u64 perf_event_time(struct perf_event *event)  in perf_event_time()  argument
1457 	struct perf_event_context *ctx = event->ctx;  in perf_event_time()
1459 	if (is_cgroup_event(event))  in perf_event_time()
1460 		return perf_cgroup_event_time(event);  in perf_event_time()
1465 static enum event_type_t get_event_type(struct perf_event *event)  in get_event_type()  argument
1467 	struct perf_event_context *ctx = event->ctx;  in get_event_type()
1476 	if (event->group_leader != event)  in get_event_type()
1477 		event = event->group_leader;  in get_event_type()
1479 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;  in get_event_type()
1489 static void init_event_group(struct perf_event *event)  in init_event_group()  argument
1491 	RB_CLEAR_NODE(&event->group_node);  in init_event_group()
1492 	event->group_index = 0;  in init_event_group()
1500 get_event_groups(struct perf_event *event, struct perf_event_context *ctx)  in get_event_groups()  argument
1502 	if (event->attr.pinned)  in get_event_groups()
1546 			 struct perf_event *event)  in perf_event_groups_insert()  argument
1552 	event->group_index = ++groups->index;  in perf_event_groups_insert()
1561 		if (perf_event_groups_less(event, node_event))  in perf_event_groups_insert()
1567 	rb_link_node(&event->group_node, parent, node);  in perf_event_groups_insert()
1568 	rb_insert_color(&event->group_node, &groups->tree);  in perf_event_groups_insert()
1575 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)  in add_event_to_groups()  argument
1579 	groups = get_event_groups(event, ctx);  in add_event_to_groups()
1580 	perf_event_groups_insert(groups, event);  in add_event_to_groups()
1588 			 struct perf_event *event)  in perf_event_groups_delete()  argument
1590 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||  in perf_event_groups_delete()
1593 	rb_erase(&event->group_node, &groups->tree);  in perf_event_groups_delete()
1594 	init_event_group(event);  in perf_event_groups_delete()
1601 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)  in del_event_from_groups()  argument
1605 	groups = get_event_groups(event, ctx);  in del_event_from_groups()
1606 	perf_event_groups_delete(groups, event);  in del_event_from_groups()
1638 perf_event_groups_next(struct perf_event *event)  in perf_event_groups_next()  argument
1642 	next = rb_entry_safe(rb_next(&event->group_node), typeof(*event), group_node);  in perf_event_groups_next()
1643 	if (next && next->cpu == event->cpu)  in perf_event_groups_next()
1652 #define perf_event_groups_for_each(event, groups)			\  argument
1653 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1654 				typeof(*event), group_node); event;	\
1655 		event = rb_entry_safe(rb_next(&event->group_node),	\
1656 				typeof(*event), group_node))
1663 list_add_event(struct perf_event *event, struct perf_event_context *ctx)  in list_add_event()  argument
1667 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in list_add_event()
1668 	event->attach_state |= PERF_ATTACH_CONTEXT;  in list_add_event()
1670 	event->tstamp = perf_event_time(event);  in list_add_event()
1677 	if (event->group_leader == event) {  in list_add_event()
1678 		event->group_caps = event->event_caps;  in list_add_event()
1679 		add_event_to_groups(event, ctx);  in list_add_event()
1682 	list_update_cgroup_event(event, ctx, true);  in list_add_event()
1684 	list_add_rcu(&event->event_entry, &ctx->event_list);  in list_add_event()
1686 	if (event->attr.inherit_stat)  in list_add_event()
1695 static inline void perf_event__state_init(struct perf_event *event)  in perf_event__state_init()  argument
1697 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :  in perf_event__state_init()
1701 static void __perf_event_read_size(struct perf_event *event, int nr_siblings)  in __perf_event_read_size()  argument
1707 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)  in __perf_event_read_size()
1710 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)  in __perf_event_read_size()
1713 	if (event->attr.read_format & PERF_FORMAT_ID)  in __perf_event_read_size()
1716 	if (event->attr.read_format & PERF_FORMAT_GROUP) {  in __perf_event_read_size()
1722 	event->read_size = size;  in __perf_event_read_size()
1725 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)  in __perf_event_header_size()  argument
1743 		size += event->read_size;  in __perf_event_header_size()
1754 	event->header_size = size;  in __perf_event_header_size()
1761 static void perf_event__header_size(struct perf_event *event)  in perf_event__header_size()  argument
1763 	__perf_event_read_size(event,  in perf_event__header_size()
1764 			       event->group_leader->nr_siblings);  in perf_event__header_size()
1765 	__perf_event_header_size(event, event->attr.sample_type);  in perf_event__header_size()
1768 static void perf_event__id_header_size(struct perf_event *event)  in perf_event__id_header_size()  argument
1771 	u64 sample_type = event->attr.sample_type;  in perf_event__id_header_size()
1792 	event->id_header_size = size;  in perf_event__id_header_size()
1795 static bool perf_event_validate_size(struct perf_event *event)  in perf_event_validate_size()  argument
1801 	__perf_event_read_size(event, event->group_leader->nr_siblings + 1);  in perf_event_validate_size()
1802 	__perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);  in perf_event_validate_size()
1803 	perf_event__id_header_size(event);  in perf_event_validate_size()
1809 	if (event->read_size + event->header_size +  in perf_event_validate_size()
1810 	    event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)  in perf_event_validate_size()
1816 static void perf_group_attach(struct perf_event *event)  in perf_group_attach()  argument
1818 	struct perf_event *group_leader = event->group_leader, *pos;  in perf_group_attach()
1820 	lockdep_assert_held(&event->ctx->lock);  in perf_group_attach()
1825 	if (event->attach_state & PERF_ATTACH_GROUP)  in perf_group_attach()
1828 	event->attach_state |= PERF_ATTACH_GROUP;  in perf_group_attach()
1830 	if (group_leader == event)  in perf_group_attach()
1833 	WARN_ON_ONCE(group_leader->ctx != event->ctx);  in perf_group_attach()
1835 	group_leader->group_caps &= event->event_caps;  in perf_group_attach()
1837 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);  in perf_group_attach()
1851 list_del_event(struct perf_event *event, struct perf_event_context *ctx)  in list_del_event()  argument
1853 	WARN_ON_ONCE(event->ctx != ctx);  in list_del_event()
1859 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))  in list_del_event()
1862 	event->attach_state &= ~PERF_ATTACH_CONTEXT;  in list_del_event()
1864 	list_update_cgroup_event(event, ctx, false);  in list_del_event()
1867 	if (event->attr.inherit_stat)  in list_del_event()
1870 	list_del_rcu(&event->event_entry);  in list_del_event()
1872 	if (event->group_leader == event)  in list_del_event()
1873 		del_event_from_groups(event, ctx);  in list_del_event()
1882 	if (event->state > PERF_EVENT_STATE_OFF)  in list_del_event()
1883 		perf_event_set_state(event, PERF_EVENT_STATE_OFF);  in list_del_event()
1888 static void perf_group_detach(struct perf_event *event)  in perf_group_detach()  argument
1891 	struct perf_event_context *ctx = event->ctx;  in perf_group_detach()
1898 	if (!(event->attach_state & PERF_ATTACH_GROUP))  in perf_group_detach()
1901 	event->attach_state &= ~PERF_ATTACH_GROUP;  in perf_group_detach()
1906 	if (event->group_leader != event) {  in perf_group_detach()
1907 		list_del_init(&event->sibling_list);  in perf_group_detach()
1908 		event->group_leader->nr_siblings--;  in perf_group_detach()
1917 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {  in perf_group_detach()
1923 		sibling->group_caps = event->group_caps;  in perf_group_detach()
1925 		if (!RB_EMPTY_NODE(&event->group_node)) {  in perf_group_detach()
1926 			add_event_to_groups(sibling, event->ctx);  in perf_group_detach()
1936 		WARN_ON_ONCE(sibling->ctx != event->ctx);  in perf_group_detach()
1940 	perf_event__header_size(event->group_leader);  in perf_group_detach()
1942 	for_each_sibling_event(tmp, event->group_leader)  in perf_group_detach()
1946 static bool is_orphaned_event(struct perf_event *event)  in is_orphaned_event()  argument
1948 	return event->state == PERF_EVENT_STATE_DEAD;  in is_orphaned_event()
1951 static inline int __pmu_filter_match(struct perf_event *event)  in __pmu_filter_match()  argument
1953 	struct pmu *pmu = event->pmu;  in __pmu_filter_match()
1954 	return pmu->filter_match ? pmu->filter_match(event) : 1;  in __pmu_filter_match()
1963 static inline int pmu_filter_match(struct perf_event *event)  in pmu_filter_match()  argument
1967 	if (!__pmu_filter_match(event))  in pmu_filter_match()
1970 	for_each_sibling_event(sibling, event) {  in pmu_filter_match()
1979 event_filter_match(struct perf_event *event)  in event_filter_match()  argument
1981 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&  in event_filter_match()
1982 	       perf_cgroup_match(event) && pmu_filter_match(event);  in event_filter_match()
1986 event_sched_out(struct perf_event *event,  in event_sched_out()  argument
1992 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_out()
1995 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in event_sched_out()
2003 	list_del_init(&event->active_list);  in event_sched_out()
2005 	perf_pmu_disable(event->pmu);  in event_sched_out()
2007 	event->pmu->del(event, 0);  in event_sched_out()
2008 	event->oncpu = -1;  in event_sched_out()
2010 	if (event->pending_disable) {  in event_sched_out()
2011 		event->pending_disable = 0;  in event_sched_out()
2014 	perf_event_set_state(event, state);  in event_sched_out()
2016 	if (!is_software_event(event))  in event_sched_out()
2020 	if (event->attr.freq && event->attr.sample_freq)  in event_sched_out()
2022 	if (event->attr.exclusive || !cpuctx->active_oncpu)  in event_sched_out()
2025 	perf_pmu_enable(event->pmu);  in event_sched_out()
2033 	struct perf_event *event;  in group_sched_out()  local
2045 	for_each_sibling_event(event, group_event)  in group_sched_out()
2046 		event_sched_out(event, cpuctx, ctx);  in group_sched_out()
2063 __perf_remove_from_context(struct perf_event *event,  in __perf_remove_from_context()  argument
2075 	event_sched_out(event, cpuctx, ctx);  in __perf_remove_from_context()
2077 		perf_group_detach(event);  in __perf_remove_from_context()
2078 	list_del_event(event, ctx);  in __perf_remove_from_context()
2099 static void perf_remove_from_context(struct perf_event *event, unsigned long flags)  in perf_remove_from_context()  argument
2101 	struct perf_event_context *ctx = event->ctx;  in perf_remove_from_context()
2105 	event_function_call(event, __perf_remove_from_context, (void *)flags);  in perf_remove_from_context()
2113 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in perf_remove_from_context()
2115 	    (event->attach_state & PERF_ATTACH_GROUP)) {  in perf_remove_from_context()
2121 		perf_group_detach(event);  in perf_remove_from_context()
2129 static void __perf_event_disable(struct perf_event *event,  in __perf_event_disable()  argument
2134 	if (event->state < PERF_EVENT_STATE_INACTIVE)  in __perf_event_disable()
2139 		update_cgrp_time_from_event(event);  in __perf_event_disable()
2142 	if (event == event->group_leader)  in __perf_event_disable()
2143 		group_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2145 		event_sched_out(event, cpuctx, ctx);  in __perf_event_disable()
2147 	perf_event_set_state(event, PERF_EVENT_STATE_OFF);  in __perf_event_disable()
2164 static void _perf_event_disable(struct perf_event *event)  in _perf_event_disable()  argument
2166 	struct perf_event_context *ctx = event->ctx;  in _perf_event_disable()
2169 	if (event->state <= PERF_EVENT_STATE_OFF) {  in _perf_event_disable()
2175 	event_function_call(event, __perf_event_disable, NULL);  in _perf_event_disable()
2178 void perf_event_disable_local(struct perf_event *event)  in perf_event_disable_local()  argument
2180 	event_function_local(event, __perf_event_disable, NULL);  in perf_event_disable_local()
2187 void perf_event_disable(struct perf_event *event)  in perf_event_disable()  argument
2191 	ctx = perf_event_ctx_lock(event);  in perf_event_disable()
2192 	_perf_event_disable(event);  in perf_event_disable()
2193 	perf_event_ctx_unlock(event, ctx);  in perf_event_disable()
2197 void perf_event_disable_inatomic(struct perf_event *event)  in perf_event_disable_inatomic()  argument
2199 	event->pending_disable = 1;  in perf_event_disable_inatomic()
2200 	irq_work_queue(&event->pending);  in perf_event_disable_inatomic()
2203 static void perf_set_shadow_time(struct perf_event *event,  in perf_set_shadow_time()  argument
2231 	if (is_cgroup_event(event))  in perf_set_shadow_time()
2232 		perf_cgroup_set_shadow_time(event, event->tstamp);  in perf_set_shadow_time()
2234 		event->shadow_ctx_time = event->tstamp - ctx->timestamp;  in perf_set_shadow_time()
2239 static void perf_log_throttle(struct perf_event *event, int enable);
2240 static void perf_log_itrace_start(struct perf_event *event);
2243 event_sched_in(struct perf_event *event,  in event_sched_in()  argument
2251 	if (event->state <= PERF_EVENT_STATE_OFF)  in event_sched_in()
2254 	WRITE_ONCE(event->oncpu, smp_processor_id());  in event_sched_in()
2261 	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);  in event_sched_in()
2268 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {  in event_sched_in()
2269 		perf_log_throttle(event, 1);  in event_sched_in()
2270 		event->hw.interrupts = 0;  in event_sched_in()
2273 	perf_pmu_disable(event->pmu);  in event_sched_in()
2275 	perf_set_shadow_time(event, ctx);  in event_sched_in()
2277 	perf_log_itrace_start(event);  in event_sched_in()
2279 	if (event->pmu->add(event, PERF_EF_START)) {  in event_sched_in()
2280 		perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in event_sched_in()
2281 		event->oncpu = -1;  in event_sched_in()
2286 	if (!is_software_event(event))  in event_sched_in()
2290 	if (event->attr.freq && event->attr.sample_freq)  in event_sched_in()
2293 	if (event->attr.exclusive)  in event_sched_in()
2297 	perf_pmu_enable(event->pmu);  in event_sched_in()
2307 	struct perf_event *event, *partial_group = NULL;  in group_sched_in()  local
2324 	for_each_sibling_event(event, group_event) {  in group_sched_in()
2325 		if (event_sched_in(event, cpuctx, ctx)) {  in group_sched_in()
2326 			partial_group = event;  in group_sched_in()
2340 	for_each_sibling_event(event, group_event) {  in group_sched_in()
2341 		if (event == partial_group)  in group_sched_in()
2344 		event_sched_out(event, cpuctx, ctx);  in group_sched_in()
2358 static int group_can_go_on(struct perf_event *event,  in group_can_go_on()  argument
2365 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)  in group_can_go_on()
2377 	if (event->attr.exclusive && cpuctx->active_oncpu)  in group_can_go_on()
2386 static void add_event_to_ctx(struct perf_event *event,  in add_event_to_ctx()  argument
2389 	list_add_event(event, ctx);  in add_event_to_ctx()
2390 	perf_group_attach(event);  in add_event_to_ctx()
2486 	struct perf_event *event = info;  in __perf_install_in_context()  local
2487 	struct perf_event_context *ctx = event->ctx;  in __perf_install_in_context()
2518 	if (is_cgroup_event(event)) {  in __perf_install_in_context()
2525 					event->cgrp->css.cgroup);  in __perf_install_in_context()
2531 		add_event_to_ctx(event, ctx);  in __perf_install_in_context()
2532 		ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_install_in_context()
2534 		add_event_to_ctx(event, ctx);  in __perf_install_in_context()
2550 			struct perf_event *event,  in perf_install_in_context()  argument
2557 	if (event->cpu != -1)  in perf_install_in_context()
2558 		event->cpu = cpu;  in perf_install_in_context()
2564 	smp_store_release(&event->ctx, ctx);  in perf_install_in_context()
2567 		cpu_function_call(cpu, __perf_install_in_context, event);  in perf_install_in_context()
2609 	if (!task_function_call(task, __perf_install_in_context, event))  in perf_install_in_context()
2631 	add_event_to_ctx(event, ctx);  in perf_install_in_context()
2638 static void __perf_event_enable(struct perf_event *event,  in __perf_event_enable()  argument
2643 	struct perf_event *leader = event->group_leader;  in __perf_event_enable()
2646 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in __perf_event_enable()
2647 	    event->state <= PERF_EVENT_STATE_ERROR)  in __perf_event_enable()
2653 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in __perf_event_enable()
2658 	if (!event_filter_match(event)) {  in __perf_event_enable()
2667 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {  in __perf_event_enable()
2676 	ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_event_enable()
2688 static void _perf_event_enable(struct perf_event *event)  in _perf_event_enable()  argument
2690 	struct perf_event_context *ctx = event->ctx;  in _perf_event_enable()
2693 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in _perf_event_enable()
2694 	    event->state <  PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
2706 	if (event->state == PERF_EVENT_STATE_ERROR)  in _perf_event_enable()
2707 		event->state = PERF_EVENT_STATE_OFF;  in _perf_event_enable()
2710 	event_function_call(event, __perf_event_enable, NULL);  in _perf_event_enable()
2716 void perf_event_enable(struct perf_event *event)  in perf_event_enable()  argument
2720 	ctx = perf_event_ctx_lock(event);  in perf_event_enable()
2721 	_perf_event_enable(event);  in perf_event_enable()
2722 	perf_event_ctx_unlock(event, ctx);  in perf_event_enable()
2727 	struct perf_event	*event;  member
2734 	struct perf_event *event = sd->event;  in __perf_event_stop()  local
2737 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in __perf_event_stop()
2747 	if (READ_ONCE(event->oncpu) != smp_processor_id())  in __perf_event_stop()
2750 	event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_stop()
2762 		event->pmu->start(event, 0);  in __perf_event_stop()
2767 static int perf_event_stop(struct perf_event *event, int restart)  in perf_event_stop()  argument
2770 		.event		= event,  in perf_event_stop()
2776 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in perf_event_stop()
2787 		ret = cpu_function_call(READ_ONCE(event->oncpu),  in perf_event_stop()
2816 void perf_event_addr_filters_sync(struct perf_event *event)  in perf_event_addr_filters_sync()  argument
2818 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_sync()
2820 	if (!has_addr_filter(event))  in perf_event_addr_filters_sync()
2824 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {  in perf_event_addr_filters_sync()
2825 		event->pmu->addr_filters_sync(event);  in perf_event_addr_filters_sync()
2826 		event->hw.addr_filters_gen = event->addr_filters_gen;  in perf_event_addr_filters_sync()
2832 static int _perf_event_refresh(struct perf_event *event, int refresh)  in _perf_event_refresh()  argument
2837 	if (event->attr.inherit || !is_sampling_event(event))  in _perf_event_refresh()
2840 	atomic_add(refresh, &event->event_limit);  in _perf_event_refresh()
2841 	_perf_event_enable(event);  in _perf_event_refresh()
2849 int perf_event_refresh(struct perf_event *event, int refresh)  in perf_event_refresh()  argument
2854 	ctx = perf_event_ctx_lock(event);  in perf_event_refresh()
2855 	ret = _perf_event_refresh(event, refresh);  in perf_event_refresh()
2856 	perf_event_ctx_unlock(event, ctx);  in perf_event_refresh()
2877 static int perf_event_modify_attr(struct perf_event *event,  in perf_event_modify_attr()  argument
2880 	if (event->attr.type != attr->type)  in perf_event_modify_attr()
2883 	switch (event->attr.type) {  in perf_event_modify_attr()
2885 		return perf_event_modify_breakpoint(event, attr);  in perf_event_modify_attr()
2896 	struct perf_event *event, *tmp;  in ctx_sched_out()  local
2944 		list_for_each_entry_safe(event, tmp, &ctx->pinned_active, active_list)  in ctx_sched_out()
2945 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
2949 		list_for_each_entry_safe(event, tmp, &ctx->flexible_active, active_list)  in ctx_sched_out()
2950 			group_sched_out(event, cpuctx, ctx);  in ctx_sched_out()
2993 static void __perf_event_sync_stat(struct perf_event *event,  in __perf_event_sync_stat()  argument
2998 	if (!event->attr.inherit_stat)  in __perf_event_sync_stat()
3008 	if (event->state == PERF_EVENT_STATE_ACTIVE)  in __perf_event_sync_stat()
3009 		event->pmu->read(event);  in __perf_event_sync_stat()
3011 	perf_event_update_time(event);  in __perf_event_sync_stat()
3018 	value = local64_xchg(&event->count, value);  in __perf_event_sync_stat()
3021 	swap(event->total_time_enabled, next_event->total_time_enabled);  in __perf_event_sync_stat()
3022 	swap(event->total_time_running, next_event->total_time_running);  in __perf_event_sync_stat()
3027 	perf_event_update_userpage(event);  in __perf_event_sync_stat()
3034 	struct perf_event *event, *next_event;  in perf_event_sync_stat()  local
3041 	event = list_first_entry(&ctx->event_list,  in perf_event_sync_stat()
3047 	while (&event->event_entry != &ctx->event_list &&  in perf_event_sync_stat()
3050 		__perf_event_sync_stat(event, next_event);  in perf_event_sync_stat()
3052 		event = list_next_entry(event, event_entry);  in perf_event_sync_stat()
3273 static int pinned_sched_in(struct perf_event *event, void *data)  in pinned_sched_in()  argument
3277 	if (event->state <= PERF_EVENT_STATE_OFF)  in pinned_sched_in()
3280 	if (!event_filter_match(event))  in pinned_sched_in()
3283 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in pinned_sched_in()
3284 		if (!group_sched_in(event, sid->cpuctx, sid->ctx))  in pinned_sched_in()
3285 			list_add_tail(&event->active_list, &sid->ctx->pinned_active);  in pinned_sched_in()
3292 	if (event->state == PERF_EVENT_STATE_INACTIVE)  in pinned_sched_in()
3293 		perf_event_set_state(event, PERF_EVENT_STATE_ERROR);  in pinned_sched_in()
3298 static int flexible_sched_in(struct perf_event *event, void *data)  in flexible_sched_in()  argument
3302 	if (event->state <= PERF_EVENT_STATE_OFF)  in flexible_sched_in()
3305 	if (!event_filter_match(event))  in flexible_sched_in()
3308 	if (group_can_go_on(event, sid->cpuctx, sid->can_add_hw)) {  in flexible_sched_in()
3309 		if (!group_sched_in(event, sid->cpuctx, sid->ctx))  in flexible_sched_in()
3310 			list_add_tail(&event->active_list, &sid->ctx->flexible_active);  in flexible_sched_in()
3477 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)  in perf_calculate_period()  argument
3479 	u64 frequency = event->attr.sample_freq;  in perf_calculate_period()
3553 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)  in perf_adjust_period()  argument
3555 	struct hw_perf_event *hwc = &event->hw;  in perf_adjust_period()
3559 	period = perf_calculate_period(event, nsec, count);  in perf_adjust_period()
3573 			event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_period()
3578 			event->pmu->start(event, PERF_EF_RELOAD);  in perf_adjust_period()
3590 	struct perf_event *event;  in perf_adjust_freq_unthr_context()  local
3606 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_adjust_freq_unthr_context()
3607 		if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_adjust_freq_unthr_context()
3610 		if (!event_filter_match(event))  in perf_adjust_freq_unthr_context()
3613 		perf_pmu_disable(event->pmu);  in perf_adjust_freq_unthr_context()
3615 		hwc = &event->hw;  in perf_adjust_freq_unthr_context()
3619 			perf_log_throttle(event, 1);  in perf_adjust_freq_unthr_context()
3620 			event->pmu->start(event, 0);  in perf_adjust_freq_unthr_context()
3623 		if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_context()
3629 		event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_freq_unthr_context()
3631 		now = local64_read(&event->count);  in perf_adjust_freq_unthr_context()
3643 			perf_adjust_period(event, period, delta, false);  in perf_adjust_freq_unthr_context()
3645 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);  in perf_adjust_freq_unthr_context()
3647 		perf_pmu_enable(event->pmu);  in perf_adjust_freq_unthr_context()
3657 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)  in rotate_ctx()  argument
3666 	perf_event_groups_delete(&ctx->flexible_groups, event);  in rotate_ctx()
3667 	perf_event_groups_insert(&ctx->flexible_groups, event);  in rotate_ctx()
3748 static int event_enable_on_exec(struct perf_event *event,  in event_enable_on_exec()  argument
3751 	if (!event->attr.enable_on_exec)  in event_enable_on_exec()
3754 	event->attr.enable_on_exec = 0;  in event_enable_on_exec()
3755 	if (event->state >= PERF_EVENT_STATE_INACTIVE)  in event_enable_on_exec()
3758 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in event_enable_on_exec()
3772 	struct perf_event *event;  in perf_event_enable_on_exec()  local
3784 	list_for_each_entry(event, &ctx->event_list, event_entry) {  in perf_event_enable_on_exec()
3785 		enabled |= event_enable_on_exec(event, ctx);  in perf_event_enable_on_exec()
3786 		event_type |= get_event_type(event);  in perf_event_enable_on_exec()
3808 	struct perf_event *event;  member
3813 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)  in __perf_event_read_cpu()  argument
3817 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {  in __perf_event_read_cpu()
3836 	struct perf_event *sub, *event = data->event;  in __perf_event_read()  local
3837 	struct perf_event_context *ctx = event->ctx;  in __perf_event_read()
3839 	struct pmu *pmu = event->pmu;  in __perf_event_read()
3854 		update_cgrp_time_from_event(event);  in __perf_event_read()
3857 	perf_event_update_time(event);  in __perf_event_read()
3859 		perf_event_update_sibling_time(event);  in __perf_event_read()
3861 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_read()
3865 		pmu->read(event);  in __perf_event_read()
3872 	pmu->read(event);  in __perf_event_read()
3874 	for_each_sibling_event(sub, event) {  in __perf_event_read()
3890 static inline u64 perf_event_count(struct perf_event *event)  in perf_event_count()  argument
3892 	return local64_read(&event->count) + atomic64_read(&event->child_count);  in perf_event_count()
3903 int perf_event_read_local(struct perf_event *event, u64 *value,  in perf_event_read_local()  argument
3919 	if (event->attr.inherit) {  in perf_event_read_local()
3925 	if ((event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
3926 	    event->hw.target != current) {  in perf_event_read_local()
3932 	if (!(event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
3933 	    event->cpu != smp_processor_id()) {  in perf_event_read_local()
3939 	if (event->attr.pinned && event->oncpu != smp_processor_id()) {  in perf_event_read_local()
3949 	if (event->oncpu == smp_processor_id())  in perf_event_read_local()
3950 		event->pmu->read(event);  in perf_event_read_local()
3952 	*value = local64_read(&event->count);  in perf_event_read_local()
3954 		u64 now = event->shadow_ctx_time + perf_clock();  in perf_event_read_local()
3957 		__perf_update_times(event, now, &__enabled, &__running);  in perf_event_read_local()
3969 static int perf_event_read(struct perf_event *event, bool group)  in perf_event_read()  argument
3971 	enum perf_event_state state = READ_ONCE(event->state);  in perf_event_read()
3990 		event_cpu = READ_ONCE(event->oncpu);  in perf_event_read()
3995 			.event = event,  in perf_event_read()
4001 		event_cpu = __perf_event_read_cpu(event, event_cpu);  in perf_event_read()
4018 		struct perf_event_context *ctx = event->ctx;  in perf_event_read()
4022 		state = event->state;  in perf_event_read()
4034 			update_cgrp_time_from_event(event);  in perf_event_read()
4037 		perf_event_update_time(event);  in perf_event_read()
4039 			perf_event_update_sibling_time(event);  in perf_event_read()
4106 		struct perf_event *event)  in find_get_context()  argument
4113 	int cpu = event->cpu;  in find_get_context()
4133 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {  in find_get_context()
4200 static void perf_event_free_filter(struct perf_event *event);
4201 static void perf_event_free_bpf_prog(struct perf_event *event);
4205 	struct perf_event *event;  in free_event_rcu()  local
4207 	event = container_of(head, struct perf_event, rcu_head);  in free_event_rcu()
4208 	if (event->ns)  in free_event_rcu()
4209 		put_pid_ns(event->ns);  in free_event_rcu()
4210 	perf_event_free_filter(event);  in free_event_rcu()
4211 	kfree(event);  in free_event_rcu()
4214 static void ring_buffer_attach(struct perf_event *event,
4217 static void detach_sb_event(struct perf_event *event)  in detach_sb_event()  argument
4219 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in detach_sb_event()
4222 	list_del_rcu(&event->sb_list);  in detach_sb_event()
4226 static bool is_sb_event(struct perf_event *event)  in is_sb_event()  argument
4228 	struct perf_event_attr *attr = &event->attr;  in is_sb_event()
4230 	if (event->parent)  in is_sb_event()
4233 	if (event->attach_state & PERF_ATTACH_TASK)  in is_sb_event()
4244 static void unaccount_pmu_sb_event(struct perf_event *event)  in unaccount_pmu_sb_event()  argument
4246 	if (is_sb_event(event))  in unaccount_pmu_sb_event()
4247 		detach_sb_event(event);  in unaccount_pmu_sb_event()
4250 static void unaccount_event_cpu(struct perf_event *event, int cpu)  in unaccount_event_cpu()  argument
4252 	if (event->parent)  in unaccount_event_cpu()
4255 	if (is_cgroup_event(event))  in unaccount_event_cpu()
4281 static void unaccount_event(struct perf_event *event)  in unaccount_event()  argument
4285 	if (event->parent)  in unaccount_event()
4288 	if (event->attach_state & PERF_ATTACH_TASK)  in unaccount_event()
4290 	if (event->attr.mmap || event->attr.mmap_data)  in unaccount_event()
4292 	if (event->attr.comm)  in unaccount_event()
4294 	if (event->attr.namespaces)  in unaccount_event()
4296 	if (event->attr.task)  in unaccount_event()
4298 	if (event->attr.freq)  in unaccount_event()
4300 	if (event->attr.context_switch) {  in unaccount_event()
4304 	if (is_cgroup_event(event))  in unaccount_event()
4306 	if (has_branch_stack(event))  in unaccount_event()
4314 	unaccount_event_cpu(event, event->cpu);  in unaccount_event()
4316 	unaccount_pmu_sb_event(event);  in unaccount_event()
4339 static int exclusive_event_init(struct perf_event *event)  in exclusive_event_init()  argument
4341 	struct pmu *pmu = event->pmu;  in exclusive_event_init()
4359 	if (event->attach_state & PERF_ATTACH_TASK) {  in exclusive_event_init()
4370 static void exclusive_event_destroy(struct perf_event *event)  in exclusive_event_destroy()  argument
4372 	struct pmu *pmu = event->pmu;  in exclusive_event_destroy()
4378 	if (event->attach_state & PERF_ATTACH_TASK)  in exclusive_event_destroy()
4395 static bool exclusive_event_installable(struct perf_event *event,  in exclusive_event_installable()  argument
4399 	struct pmu *pmu = event->pmu;  in exclusive_event_installable()
4405 		if (exclusive_event_match(iter_event, event))  in exclusive_event_installable()
4412 static void perf_addr_filters_splice(struct perf_event *event,
4415 static void _free_event(struct perf_event *event)  in _free_event()  argument
4417 	irq_work_sync(&event->pending);  in _free_event()
4419 	unaccount_event(event);  in _free_event()
4421 	if (event->rb) {  in _free_event()
4428 		mutex_lock(&event->mmap_mutex);  in _free_event()
4429 		ring_buffer_attach(event, NULL);  in _free_event()
4430 		mutex_unlock(&event->mmap_mutex);  in _free_event()
4433 	if (is_cgroup_event(event))  in _free_event()
4434 		perf_detach_cgroup(event);  in _free_event()
4436 	if (!event->parent) {  in _free_event()
4437 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in _free_event()
4441 	perf_event_free_bpf_prog(event);  in _free_event()
4442 	perf_addr_filters_splice(event, NULL);  in _free_event()
4443 	kfree(event->addr_filters_offs);  in _free_event()
4445 	if (event->destroy)  in _free_event()
4446 		event->destroy(event);  in _free_event()
4448 	if (event->ctx)  in _free_event()
4449 		put_ctx(event->ctx);  in _free_event()
4451 	if (event->hw.target)  in _free_event()
4452 		put_task_struct(event->hw.target);  in _free_event()
4454 	exclusive_event_destroy(event);  in _free_event()
4455 	module_put(event->pmu->module);  in _free_event()
4457 	call_rcu(&event->rcu_head, free_event_rcu);  in _free_event()
4464 static void free_event(struct perf_event *event)  in free_event()  argument
4466 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,  in free_event()
4468 				atomic_long_read(&event->refcount), event)) {  in free_event()
4473 	_free_event(event);  in free_event()
4479 static void perf_remove_from_owner(struct perf_event *event)  in perf_remove_from_owner()  argument
4490 	owner = READ_ONCE(event->owner);  in perf_remove_from_owner()
4518 		if (event->owner) {  in perf_remove_from_owner()
4519 			list_del_init(&event->owner_entry);  in perf_remove_from_owner()
4520 			smp_store_release(&event->owner, NULL);  in perf_remove_from_owner()
4527 static void put_event(struct perf_event *event)  in put_event()  argument
4529 	if (!atomic_long_dec_and_test(&event->refcount))  in put_event()
4532 	_free_event(event);  in put_event()
4540 int perf_event_release_kernel(struct perf_event *event)  in perf_event_release_kernel()  argument
4542 	struct perf_event_context *ctx = event->ctx;  in perf_event_release_kernel()
4551 		WARN_ON_ONCE(event->attach_state &  in perf_event_release_kernel()
4556 	if (!is_kernel_event(event))  in perf_event_release_kernel()
4557 		perf_remove_from_owner(event);  in perf_event_release_kernel()
4559 	ctx = perf_event_ctx_lock(event);  in perf_event_release_kernel()
4561 	perf_remove_from_context(event, DETACH_GROUP);  in perf_event_release_kernel()
4575 	event->state = PERF_EVENT_STATE_DEAD;  in perf_event_release_kernel()
4578 	perf_event_ctx_unlock(event, ctx);  in perf_event_release_kernel()
4581 	mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
4582 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_release_kernel()
4604 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
4606 		mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
4613 		tmp = list_first_entry_or_null(&event->child_list,  in perf_event_release_kernel()
4622 			put_event(event);  in perf_event_release_kernel()
4625 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
4630 	mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
4638 	put_event(event); /* Must be the 'last' reference */  in perf_event_release_kernel()
4652 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)  in __perf_event_read_value()  argument
4660 	mutex_lock(&event->child_mutex);  in __perf_event_read_value()
4662 	(void)perf_event_read(event, false);  in __perf_event_read_value()
4663 	total += perf_event_count(event);  in __perf_event_read_value()
4665 	*enabled += event->total_time_enabled +  in __perf_event_read_value()
4666 			atomic64_read(&event->child_total_time_enabled);  in __perf_event_read_value()
4667 	*running += event->total_time_running +  in __perf_event_read_value()
4668 			atomic64_read(&event->child_total_time_running);  in __perf_event_read_value()
4670 	list_for_each_entry(child, &event->child_list, child_list) {  in __perf_event_read_value()
4676 	mutex_unlock(&event->child_mutex);  in __perf_event_read_value()
4681 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)  in perf_event_read_value()  argument
4686 	ctx = perf_event_ctx_lock(event);  in perf_event_read_value()
4687 	count = __perf_event_read_value(event, enabled, running);  in perf_event_read_value()
4688 	perf_event_ctx_unlock(event, ctx);  in perf_event_read_value()
4741 static int perf_read_group(struct perf_event *event,  in perf_read_group()  argument
4744 	struct perf_event *leader = event->group_leader, *child;  in perf_read_group()
4751 	values = kzalloc(event->read_size, GFP_KERNEL);  in perf_read_group()
4775 	ret = event->read_size;  in perf_read_group()
4776 	if (copy_to_user(buf, values, event->read_size))  in perf_read_group()
4787 static int perf_read_one(struct perf_event *event,  in perf_read_one()  argument
4794 	values[n++] = __perf_event_read_value(event, &enabled, &running);  in perf_read_one()
4800 		values[n++] = primary_event_id(event);  in perf_read_one()
4808 static bool is_event_hup(struct perf_event *event)  in is_event_hup()  argument
4812 	if (event->state > PERF_EVENT_STATE_EXIT)  in is_event_hup()
4815 	mutex_lock(&event->child_mutex);  in is_event_hup()
4816 	no_children = list_empty(&event->child_list);  in is_event_hup()
4817 	mutex_unlock(&event->child_mutex);  in is_event_hup()
4825 __perf_read(struct perf_event *event, char __user *buf, size_t count)  in __perf_read()  argument
4827 	u64 read_format = event->attr.read_format;  in __perf_read()
4835 	if (event->state == PERF_EVENT_STATE_ERROR)  in __perf_read()
4838 	if (count < event->read_size)  in __perf_read()
4841 	WARN_ON_ONCE(event->ctx->parent_ctx);  in __perf_read()
4843 		ret = perf_read_group(event, read_format, buf);  in __perf_read()
4845 		ret = perf_read_one(event, read_format, buf);  in __perf_read()
4853 	struct perf_event *event = file->private_data;  in perf_read()  local
4857 	ctx = perf_event_ctx_lock(event);  in perf_read()
4858 	ret = __perf_read(event, buf, count);  in perf_read()
4859 	perf_event_ctx_unlock(event, ctx);  in perf_read()
4866 	struct perf_event *event = file->private_data;  in perf_poll()  local
4870 	poll_wait(file, &event->waitq, wait);  in perf_poll()
4872 	if (is_event_hup(event))  in perf_poll()
4879 	mutex_lock(&event->mmap_mutex);  in perf_poll()
4880 	rb = event->rb;  in perf_poll()
4883 	mutex_unlock(&event->mmap_mutex);  in perf_poll()
4887 static void _perf_event_reset(struct perf_event *event)  in _perf_event_reset()  argument
4889 	(void)perf_event_read(event, false);  in _perf_event_reset()
4890 	local64_set(&event->count, 0);  in _perf_event_reset()
4891 	perf_event_update_userpage(event);  in _perf_event_reset()
4900 static void perf_event_for_each_child(struct perf_event *event,  in perf_event_for_each_child()  argument
4905 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_for_each_child()
4907 	mutex_lock(&event->child_mutex);  in perf_event_for_each_child()
4908 	func(event);  in perf_event_for_each_child()
4909 	list_for_each_entry(child, &event->child_list, child_list)  in perf_event_for_each_child()
4911 	mutex_unlock(&event->child_mutex);  in perf_event_for_each_child()
4914 static void perf_event_for_each(struct perf_event *event,  in perf_event_for_each()  argument
4917 	struct perf_event_context *ctx = event->ctx;  in perf_event_for_each()
4922 	event = event->group_leader;  in perf_event_for_each()
4924 	perf_event_for_each_child(event, func);  in perf_event_for_each()
4925 	for_each_sibling_event(sibling, event)  in perf_event_for_each()
4929 static void __perf_event_period(struct perf_event *event,  in __perf_event_period()  argument
4937 	if (event->attr.freq) {  in __perf_event_period()
4938 		event->attr.sample_freq = value;  in __perf_event_period()
4940 		event->attr.sample_period = value;  in __perf_event_period()
4941 		event->hw.sample_period = value;  in __perf_event_period()
4944 	active = (event->state == PERF_EVENT_STATE_ACTIVE);  in __perf_event_period()
4951 		if (event->hw.interrupts == MAX_INTERRUPTS) {  in __perf_event_period()
4952 			event->hw.interrupts = 0;  in __perf_event_period()
4953 			perf_log_throttle(event, 1);  in __perf_event_period()
4955 		event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_period()
4958 	local64_set(&event->hw.period_left, 0);  in __perf_event_period()
4961 		event->pmu->start(event, PERF_EF_RELOAD);  in __perf_event_period()
4966 static int perf_event_period(struct perf_event *event, u64 __user *arg)  in perf_event_period()  argument
4970 	if (!is_sampling_event(event))  in perf_event_period()
4979 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)  in perf_event_period()
4982 	event_function_call(event, __perf_event_period, &value);  in perf_event_period()
5003 static int perf_event_set_output(struct perf_event *event,
5005 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5006 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd);
5010 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)  in _perf_ioctl()  argument
5027 		return _perf_event_refresh(event, arg);  in _perf_ioctl()
5030 		return perf_event_period(event, (u64 __user *)arg);  in _perf_ioctl()
5034 		u64 id = primary_event_id(event);  in _perf_ioctl()
5051 			ret = perf_event_set_output(event, output_event);  in _perf_ioctl()
5054 			ret = perf_event_set_output(event, NULL);  in _perf_ioctl()
5060 		return perf_event_set_filter(event, (void __user *)arg);  in _perf_ioctl()
5063 		return perf_event_set_bpf_prog(event, arg);  in _perf_ioctl()
5069 		rb = rcu_dereference(event->rb);  in _perf_ioctl()
5080 		return perf_event_query_prog_array(event, (void __user *)arg);  in _perf_ioctl()
5090 		return perf_event_modify_attr(event,  &new_attr);  in _perf_ioctl()
5097 		perf_event_for_each(event, func);  in _perf_ioctl()
5099 		perf_event_for_each_child(event, func);  in _perf_ioctl()
5106 	struct perf_event *event = file->private_data;  in perf_ioctl()  local
5110 	ctx = perf_event_ctx_lock(event);  in perf_ioctl()
5111 	ret = _perf_ioctl(event, cmd, arg);  in perf_ioctl()
5112 	perf_event_ctx_unlock(event, ctx);  in perf_ioctl()
5142 	struct perf_event *event;  in perf_event_task_enable()  local
5145 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_enable()
5146 		ctx = perf_event_ctx_lock(event);  in perf_event_task_enable()
5147 		perf_event_for_each_child(event, _perf_event_enable);  in perf_event_task_enable()
5148 		perf_event_ctx_unlock(event, ctx);  in perf_event_task_enable()
5158 	struct perf_event *event;  in perf_event_task_disable()  local
5161 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_disable()
5162 		ctx = perf_event_ctx_lock(event);  in perf_event_task_disable()
5163 		perf_event_for_each_child(event, _perf_event_disable);  in perf_event_task_disable()
5164 		perf_event_ctx_unlock(event, ctx);  in perf_event_task_disable()
5171 static int perf_event_index(struct perf_event *event)  in perf_event_index()  argument
5173 	if (event->hw.state & PERF_HES_STOPPED)  in perf_event_index()
5176 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_event_index()
5179 	return event->pmu->event_idx(event);  in perf_event_index()
5182 static void calc_timer_values(struct perf_event *event,  in calc_timer_values()  argument
5190 	ctx_time = event->shadow_ctx_time + *now;  in calc_timer_values()
5191 	__perf_update_times(event, ctx_time, enabled, running);  in calc_timer_values()
5194 static void perf_event_init_userpage(struct perf_event *event)  in perf_event_init_userpage()  argument
5200 	rb = rcu_dereference(event->rb);  in perf_event_init_userpage()
5217 	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)  in arch_perf_update_userpage()  argument
5226 void perf_event_update_userpage(struct perf_event *event)  in perf_event_update_userpage()  argument
5233 	rb = rcu_dereference(event->rb);  in perf_event_update_userpage()
5246 	calc_timer_values(event, &now, &enabled, &running);  in perf_event_update_userpage()
5256 	userpg->index = perf_event_index(event);  in perf_event_update_userpage()
5257 	userpg->offset = perf_event_count(event);  in perf_event_update_userpage()
5259 		userpg->offset -= local64_read(&event->hw.prev_count);  in perf_event_update_userpage()
5262 			atomic64_read(&event->child_total_time_enabled);  in perf_event_update_userpage()
5265 			atomic64_read(&event->child_total_time_running);  in perf_event_update_userpage()
5267 	arch_perf_update_userpage(event, userpg, now);  in perf_event_update_userpage()
5279 	struct perf_event *event = vmf->vma->vm_file->private_data;  in perf_mmap_fault()  local
5290 	rb = rcu_dereference(event->rb);  in perf_mmap_fault()
5312 static void ring_buffer_attach(struct perf_event *event,  in ring_buffer_attach()  argument
5318 	if (event->rb) {  in ring_buffer_attach()
5323 		WARN_ON_ONCE(event->rcu_pending);  in ring_buffer_attach()
5325 		old_rb = event->rb;  in ring_buffer_attach()
5327 		list_del_rcu(&event->rb_entry);  in ring_buffer_attach()
5330 		event->rcu_batches = get_state_synchronize_rcu();  in ring_buffer_attach()
5331 		event->rcu_pending = 1;  in ring_buffer_attach()
5335 		if (event->rcu_pending) {  in ring_buffer_attach()
5336 			cond_synchronize_rcu(event->rcu_batches);  in ring_buffer_attach()
5337 			event->rcu_pending = 0;  in ring_buffer_attach()
5341 		list_add_rcu(&event->rb_entry, &rb->event_list);  in ring_buffer_attach()
5355 	if (has_aux(event))  in ring_buffer_attach()
5356 		perf_event_stop(event, 0);  in ring_buffer_attach()
5358 	rcu_assign_pointer(event->rb, rb);  in ring_buffer_attach()
5367 		wake_up_all(&event->waitq);  in ring_buffer_attach()
5371 static void ring_buffer_wakeup(struct perf_event *event)  in ring_buffer_wakeup()  argument
5376 	rb = rcu_dereference(event->rb);  in ring_buffer_wakeup()
5378 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)  in ring_buffer_wakeup()
5379 			wake_up_all(&event->waitq);  in ring_buffer_wakeup()
5384 struct ring_buffer *ring_buffer_get(struct perf_event *event)  in ring_buffer_get()  argument
5389 	rb = rcu_dereference(event->rb);  in ring_buffer_get()
5411 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_open()  local
5413 	atomic_inc(&event->mmap_count);  in perf_mmap_open()
5414 	atomic_inc(&event->rb->mmap_count);  in perf_mmap_open()
5417 		atomic_inc(&event->rb->aux_mmap_count);  in perf_mmap_open()
5419 	if (event->pmu->event_mapped)  in perf_mmap_open()
5420 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap_open()
5423 static void perf_pmu_output_stop(struct perf_event *event);
5435 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_close()  local
5437 	struct ring_buffer *rb = ring_buffer_get(event);  in perf_mmap_close()
5442 	if (event->pmu->event_unmapped)  in perf_mmap_close()
5443 		event->pmu->event_unmapped(event, vma->vm_mm);  in perf_mmap_close()
5451 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {  in perf_mmap_close()
5458 		perf_pmu_output_stop(event);  in perf_mmap_close()
5468 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
5473 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))  in perf_mmap_close()
5476 	ring_buffer_attach(event, NULL);  in perf_mmap_close()
5477 	mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
5490 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {  in perf_mmap_close()
5491 		if (!atomic_long_inc_not_zero(&event->refcount)) {  in perf_mmap_close()
5500 		mutex_lock(&event->mmap_mutex);  in perf_mmap_close()
5511 		if (event->rb == rb)  in perf_mmap_close()
5512 			ring_buffer_attach(event, NULL);  in perf_mmap_close()
5514 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
5515 		put_event(event);  in perf_mmap_close()
5551 	struct perf_event *event = file->private_data;  in perf_mmap()  local
5566 	if (event->cpu == -1 && event->attr.inherit)  in perf_mmap()
5584 		if (!event->rb)  in perf_mmap()
5589 		mutex_lock(&event->mmap_mutex);  in perf_mmap()
5592 		rb = event->rb;  in perf_mmap()
5644 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_mmap()
5646 	mutex_lock(&event->mmap_mutex);  in perf_mmap()
5647 	if (event->rb) {  in perf_mmap()
5648 		if (event->rb->nr_pages != nr_pages) {  in perf_mmap()
5653 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {  in perf_mmap()
5659 			mutex_unlock(&event->mmap_mutex);  in perf_mmap()
5691 	WARN_ON(!rb && event->rb);  in perf_mmap()
5698 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,  in perf_mmap()
5699 			      event->cpu, flags);  in perf_mmap()
5710 		ring_buffer_attach(event, rb);  in perf_mmap()
5712 		perf_event_init_userpage(event);  in perf_mmap()
5713 		perf_event_update_userpage(event);  in perf_mmap()
5715 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,  in perf_mmap()
5716 				   event->attr.aux_watermark, flags);  in perf_mmap()
5726 		atomic_inc(&event->mmap_count);  in perf_mmap()
5731 	mutex_unlock(&event->mmap_mutex);  in perf_mmap()
5740 	if (event->pmu->event_mapped)  in perf_mmap()
5741 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap()
5749 	struct perf_event *event = filp->private_data;  in perf_fasync()  local
5753 	retval = fasync_helper(fd, filp, on, &event->fasync);  in perf_fasync()
5780 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)  in perf_event_fasync()  argument
5783 	if (event->parent)  in perf_event_fasync()
5784 		event = event->parent;  in perf_event_fasync()
5785 	return &event->fasync;  in perf_event_fasync()
5788 void perf_event_wakeup(struct perf_event *event)  in perf_event_wakeup()  argument
5790 	ring_buffer_wakeup(event);  in perf_event_wakeup()
5792 	if (event->pending_kill) {  in perf_event_wakeup()
5793 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);  in perf_event_wakeup()
5794 		event->pending_kill = 0;  in perf_event_wakeup()
5800 	struct perf_event *event = container_of(entry,  in perf_pending_event()  local
5810 	if (event->pending_disable) {  in perf_pending_event()
5811 		event->pending_disable = 0;  in perf_pending_event()
5812 		perf_event_disable_local(event);  in perf_pending_event()
5815 	if (event->pending_wakeup) {  in perf_pending_event()
5816 		event->pending_wakeup = 0;  in perf_pending_event()
5817 		perf_event_wakeup(event);  in perf_pending_event()
5985 					 struct perf_event *event)  in __perf_event_header__init_id()  argument
5987 	u64 sample_type = event->attr.sample_type;  in __perf_event_header__init_id()
5990 	header->size += event->id_header_size;  in __perf_event_header__init_id()
5994 		data->tid_entry.pid = perf_event_pid(event, current);  in __perf_event_header__init_id()
5995 		data->tid_entry.tid = perf_event_tid(event, current);  in __perf_event_header__init_id()
5999 		data->time = perf_event_clock(event);  in __perf_event_header__init_id()
6002 		data->id = primary_event_id(event);  in __perf_event_header__init_id()
6005 		data->stream_id = event->id;  in __perf_event_header__init_id()
6015 				struct perf_event *event)  in perf_event_header__init_id()  argument
6017 	if (event->attr.sample_id_all)  in perf_event_header__init_id()
6018 		__perf_event_header__init_id(header, data, event);  in perf_event_header__init_id()
6045 void perf_event__output_id_sample(struct perf_event *event,  in perf_event__output_id_sample()  argument
6049 	if (event->attr.sample_id_all)  in perf_event__output_id_sample()
6054 				 struct perf_event *event,  in perf_output_read_one()  argument
6057 	u64 read_format = event->attr.read_format;  in perf_output_read_one()
6061 	values[n++] = perf_event_count(event);  in perf_output_read_one()
6064 			atomic64_read(&event->child_total_time_enabled);  in perf_output_read_one()
6068 			atomic64_read(&event->child_total_time_running);  in perf_output_read_one()
6071 		values[n++] = primary_event_id(event);  in perf_output_read_one()
6077 			    struct perf_event *event,  in perf_output_read_group()  argument
6080 	struct perf_event *leader = event->group_leader, *sub;  in perf_output_read_group()
6081 	u64 read_format = event->attr.read_format;  in perf_output_read_group()
6093 	if ((leader != event) &&  in perf_output_read_group()
6106 		if ((sub != event) &&  in perf_output_read_group()
6129 			     struct perf_event *event)  in perf_output_read()  argument
6132 	u64 read_format = event->attr.read_format;  in perf_output_read()
6144 		calc_timer_values(event, &now, &enabled, &running);  in perf_output_read()
6146 	if (event->attr.read_format & PERF_FORMAT_GROUP)  in perf_output_read()
6147 		perf_output_read_group(handle, event, enabled, running);  in perf_output_read()
6149 		perf_output_read_one(handle, event, enabled, running);  in perf_output_read()
6155 			struct perf_event *event)  in perf_output_sample()  argument
6189 		perf_output_read(handle, event);  in perf_output_sample()
6260 			u64 mask = event->attr.sample_regs_user;  in perf_output_sample()
6291 			u64 mask = event->attr.sample_regs_intr;  in perf_output_sample()
6302 	if (!event->attr.watermark) {  in perf_output_sample()
6303 		int wakeup_events = event->attr.wakeup_events;  in perf_output_sample()
6352 perf_callchain(struct perf_event *event, struct pt_regs *regs)  in perf_callchain()  argument
6354 	bool kernel = !event->attr.exclude_callchain_kernel;  in perf_callchain()
6355 	bool user   = !event->attr.exclude_callchain_user;  in perf_callchain()
6357 	bool crosstask = event->ctx->task && event->ctx->task != current;  in perf_callchain()
6358 	const u32 max_stack = event->attr.sample_max_stack;  in perf_callchain()
6371 			 struct perf_event *event,  in perf_prepare_sample()  argument
6374 	u64 sample_type = event->attr.sample_type;  in perf_prepare_sample()
6377 	header->size = sizeof(*header) + event->header_size;  in perf_prepare_sample()
6382 	__perf_event_header__init_id(header, data, event);  in perf_prepare_sample()
6391 			data->callchain = perf_callchain(event, regs);  in perf_prepare_sample()
6441 			u64 mask = event->attr.sample_regs_user;  in perf_prepare_sample()
6455 		u16 stack_size = event->attr.sample_stack_user;  in perf_prepare_sample()
6480 			u64 mask = event->attr.sample_regs_intr;  in perf_prepare_sample()
6493 __perf_event_output(struct perf_event *event,  in __perf_event_output()  argument
6506 	perf_prepare_sample(&header, data, event, regs);  in __perf_event_output()
6508 	if (output_begin(&handle, event, header.size))  in __perf_event_output()
6511 	perf_output_sample(&handle, &header, data, event);  in __perf_event_output()
6520 perf_event_output_forward(struct perf_event *event,  in perf_event_output_forward()  argument
6524 	__perf_event_output(event, data, regs, perf_output_begin_forward);  in perf_event_output_forward()
6528 perf_event_output_backward(struct perf_event *event,  in perf_event_output_backward()  argument
6532 	__perf_event_output(event, data, regs, perf_output_begin_backward);  in perf_event_output_backward()
6536 perf_event_output(struct perf_event *event,  in perf_event_output()  argument
6540 	__perf_event_output(event, data, regs, perf_output_begin);  in perf_event_output()
6555 perf_event_read_event(struct perf_event *event,  in perf_event_read_event()  argument
6564 			.size = sizeof(read_event) + event->read_size,  in perf_event_read_event()
6566 		.pid = perf_event_pid(event, task),  in perf_event_read_event()
6567 		.tid = perf_event_tid(event, task),  in perf_event_read_event()
6571 	perf_event_header__init_id(&read_event.header, &sample, event);  in perf_event_read_event()
6572 	ret = perf_output_begin(&handle, event, read_event.header.size);  in perf_event_read_event()
6577 	perf_output_read(&handle, event);  in perf_event_read_event()
6578 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_read_event()
6583 typedef void (perf_iterate_f)(struct perf_event *event, void *data);
6590 	struct perf_event *event;  in perf_iterate_ctx()  local
6592 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_iterate_ctx()
6594 			if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_ctx()
6596 			if (!event_filter_match(event))  in perf_iterate_ctx()
6600 		output(event, data);  in perf_iterate_ctx()
6607 	struct perf_event *event;  in perf_iterate_sb_cpu()  local
6609 	list_for_each_entry_rcu(event, &pel->list, sb_list) {  in perf_iterate_sb_cpu()
6615 		if (!smp_load_acquire(&event->ctx))  in perf_iterate_sb_cpu()
6618 		if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_sb_cpu()
6620 		if (!event_filter_match(event))  in perf_iterate_sb_cpu()
6622 		output(event, data);  in perf_iterate_sb_cpu()
6668 static void perf_event_addr_filters_exec(struct perf_event *event, void *data)  in perf_event_addr_filters_exec()  argument
6670 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_exec()
6675 	if (!has_addr_filter(event))  in perf_event_addr_filters_exec()
6681 			event->addr_filters_offs[count] = 0;  in perf_event_addr_filters_exec()
6689 		event->addr_filters_gen++;  in perf_event_addr_filters_exec()
6693 		perf_event_stop(event, 1);  in perf_event_addr_filters_exec()
6720 static void __perf_event_output_stop(struct perf_event *event, void *data)  in __perf_event_output_stop()  argument
6722 	struct perf_event *parent = event->parent;  in __perf_event_output_stop()
6726 		.event	= event,  in __perf_event_output_stop()
6729 	if (!has_aux(event))  in __perf_event_output_stop()
6733 		parent = event;  in __perf_event_output_stop()
6751 	struct perf_event *event = info;  in __perf_pmu_output_stop()  local
6752 	struct pmu *pmu = event->pmu;  in __perf_pmu_output_stop()
6755 		.rb	= event->rb,  in __perf_pmu_output_stop()
6768 static void perf_pmu_output_stop(struct perf_event *event)  in perf_pmu_output_stop()  argument
6775 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {  in perf_pmu_output_stop()
6789 		err = cpu_function_call(cpu, __perf_pmu_output_stop, event);  in perf_pmu_output_stop()
6819 static int perf_event_task_match(struct perf_event *event)  in perf_event_task_match()  argument
6821 	return event->attr.comm  || event->attr.mmap ||  in perf_event_task_match()
6822 	       event->attr.mmap2 || event->attr.mmap_data ||  in perf_event_task_match()
6823 	       event->attr.task;  in perf_event_task_match()
6826 static void perf_event_task_output(struct perf_event *event,  in perf_event_task_output()  argument
6835 	if (!perf_event_task_match(event))  in perf_event_task_output()
6838 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);  in perf_event_task_output()
6840 	ret = perf_output_begin(&handle, event,  in perf_event_task_output()
6845 	task_event->event_id.pid = perf_event_pid(event, task);  in perf_event_task_output()
6846 	task_event->event_id.ppid = perf_event_pid(event, current);  in perf_event_task_output()
6848 	task_event->event_id.tid = perf_event_tid(event, task);  in perf_event_task_output()
6849 	task_event->event_id.ptid = perf_event_tid(event, current);  in perf_event_task_output()
6851 	task_event->event_id.time = perf_event_clock(event);  in perf_event_task_output()
6855 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_task_output()
6918 static int perf_event_comm_match(struct perf_event *event)  in perf_event_comm_match()  argument
6920 	return event->attr.comm;  in perf_event_comm_match()
6923 static void perf_event_comm_output(struct perf_event *event,  in perf_event_comm_output()  argument
6932 	if (!perf_event_comm_match(event))  in perf_event_comm_output()
6935 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);  in perf_event_comm_output()
6936 	ret = perf_output_begin(&handle, event,  in perf_event_comm_output()
6942 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);  in perf_event_comm_output()
6943 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);  in perf_event_comm_output()
6949 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_comm_output()
7017 static int perf_event_namespaces_match(struct perf_event *event)  in perf_event_namespaces_match()  argument
7019 	return event->attr.namespaces;  in perf_event_namespaces_match()
7022 static void perf_event_namespaces_output(struct perf_event *event,  in perf_event_namespaces_output()  argument
7031 	if (!perf_event_namespaces_match(event))  in perf_event_namespaces_output()
7035 				   &sample, event);  in perf_event_namespaces_output()
7036 	ret = perf_output_begin(&handle, event,  in perf_event_namespaces_output()
7041 	namespaces_event->event_id.pid = perf_event_pid(event,  in perf_event_namespaces_output()
7043 	namespaces_event->event_id.tid = perf_event_tid(event,  in perf_event_namespaces_output()
7048 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_namespaces_output()
7155 static int perf_event_mmap_match(struct perf_event *event,  in perf_event_mmap_match()  argument
7162 	return (!executable && event->attr.mmap_data) ||  in perf_event_mmap_match()
7163 	       (executable && (event->attr.mmap || event->attr.mmap2));  in perf_event_mmap_match()
7166 static void perf_event_mmap_output(struct perf_event *event,  in perf_event_mmap_output()  argument
7175 	if (!perf_event_mmap_match(event, data))  in perf_event_mmap_output()
7178 	if (event->attr.mmap2) {  in perf_event_mmap_output()
7188 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);  in perf_event_mmap_output()
7189 	ret = perf_output_begin(&handle, event,  in perf_event_mmap_output()
7194 	mmap_event->event_id.pid = perf_event_pid(event, current);  in perf_event_mmap_output()
7195 	mmap_event->event_id.tid = perf_event_tid(event, current);  in perf_event_mmap_output()
7199 	if (event->attr.mmap2) {  in perf_event_mmap_output()
7211 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_mmap_output()
7361 static void __perf_addr_filters_adjust(struct perf_event *event, void *data)  in __perf_addr_filters_adjust()  argument
7363 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in __perf_addr_filters_adjust()
7370 	if (!has_addr_filter(event))  in __perf_addr_filters_adjust()
7380 			event->addr_filters_offs[count] = vma->vm_start;  in __perf_addr_filters_adjust()
7388 		event->addr_filters_gen++;  in __perf_addr_filters_adjust()
7392 		perf_event_stop(event, 1);  in __perf_addr_filters_adjust()
7456 void perf_event_aux_event(struct perf_event *event, unsigned long head,  in perf_event_aux_event()  argument
7478 	perf_event_header__init_id(&rec.header, &sample, event);  in perf_event_aux_event()
7479 	ret = perf_output_begin(&handle, event, rec.header.size);  in perf_event_aux_event()
7485 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_aux_event()
7493 void perf_log_lost_samples(struct perf_event *event, u64 lost)  in perf_log_lost_samples()  argument
7511 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);  in perf_log_lost_samples()
7513 	ret = perf_output_begin(&handle, event,  in perf_log_lost_samples()
7519 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_lost_samples()
7538 static int perf_event_switch_match(struct perf_event *event)  in perf_event_switch_match()  argument
7540 	return event->attr.context_switch;  in perf_event_switch_match()
7543 static void perf_event_switch_output(struct perf_event *event, void *data)  in perf_event_switch_output()  argument
7550 	if (!perf_event_switch_match(event))  in perf_event_switch_output()
7554 	if (event->ctx->task) {  in perf_event_switch_output()
7561 					perf_event_pid(event, se->next_prev);  in perf_event_switch_output()
7563 					perf_event_tid(event, se->next_prev);  in perf_event_switch_output()
7566 	perf_event_header__init_id(&se->event_id.header, &sample, event);  in perf_event_switch_output()
7568 	ret = perf_output_begin(&handle, event, se->event_id.header.size);  in perf_event_switch_output()
7572 	if (event->ctx->task)  in perf_event_switch_output()
7577 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_switch_output()
7616 static void perf_log_throttle(struct perf_event *event, int enable)  in perf_log_throttle()  argument
7633 		.time		= perf_event_clock(event),  in perf_log_throttle()
7634 		.id		= primary_event_id(event),  in perf_log_throttle()
7635 		.stream_id	= event->id,  in perf_log_throttle()
7641 	perf_event_header__init_id(&throttle_event.header, &sample, event);  in perf_log_throttle()
7643 	ret = perf_output_begin(&handle, event,  in perf_log_throttle()
7649 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_throttle()
7653 void perf_event_itrace_started(struct perf_event *event)  in perf_event_itrace_started()  argument
7655 	event->attach_state |= PERF_ATTACH_ITRACE;  in perf_event_itrace_started()
7658 static void perf_log_itrace_start(struct perf_event *event)  in perf_log_itrace_start()  argument
7669 	if (event->parent)  in perf_log_itrace_start()
7670 		event = event->parent;  in perf_log_itrace_start()
7672 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||  in perf_log_itrace_start()
7673 	    event->attach_state & PERF_ATTACH_ITRACE)  in perf_log_itrace_start()
7679 	rec.pid	= perf_event_pid(event, current);  in perf_log_itrace_start()
7680 	rec.tid	= perf_event_tid(event, current);  in perf_log_itrace_start()
7682 	perf_event_header__init_id(&rec.header, &sample, event);  in perf_log_itrace_start()
7683 	ret = perf_output_begin(&handle, event, rec.header.size);  in perf_log_itrace_start()
7689 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_itrace_start()
7695 __perf_event_account_interrupt(struct perf_event *event, int throttle)  in __perf_event_account_interrupt()  argument
7697 	struct hw_perf_event *hwc = &event->hw;  in __perf_event_account_interrupt()
7712 			perf_log_throttle(event, 0);  in __perf_event_account_interrupt()
7717 	if (event->attr.freq) {  in __perf_event_account_interrupt()
7724 			perf_adjust_period(event, delta, hwc->last_period, true);  in __perf_event_account_interrupt()
7730 int perf_event_account_interrupt(struct perf_event *event)  in perf_event_account_interrupt()  argument
7732 	return __perf_event_account_interrupt(event, 1);  in perf_event_account_interrupt()
7739 static int __perf_event_overflow(struct perf_event *event,  in __perf_event_overflow()  argument
7743 	int events = atomic_read(&event->event_limit);  in __perf_event_overflow()
7750 	if (unlikely(!is_sampling_event(event)))  in __perf_event_overflow()
7753 	ret = __perf_event_account_interrupt(event, throttle);  in __perf_event_overflow()
7760 	event->pending_kill = POLL_IN;  in __perf_event_overflow()
7761 	if (events && atomic_dec_and_test(&event->event_limit)) {  in __perf_event_overflow()
7763 		event->pending_kill = POLL_HUP;  in __perf_event_overflow()
7765 		perf_event_disable_inatomic(event);  in __perf_event_overflow()
7768 	READ_ONCE(event->overflow_handler)(event, data, regs);  in __perf_event_overflow()  local
7770 	if (*perf_event_fasync(event) && event->pending_kill) {  in __perf_event_overflow()
7771 		event->pending_wakeup = 1;  in __perf_event_overflow()
7772 		irq_work_queue(&event->pending);  in __perf_event_overflow()
7778 int perf_event_overflow(struct perf_event *event,  in perf_event_overflow()  argument
7782 	return __perf_event_overflow(event, 1, data, regs);  in perf_event_overflow()
7807 u64 perf_swevent_set_period(struct perf_event *event)  in perf_swevent_set_period()  argument
7809 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_set_period()
7830 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,  in perf_swevent_overflow()  argument
7834 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_overflow()
7838 		overflow = perf_swevent_set_period(event);  in perf_swevent_overflow()
7844 		if (__perf_event_overflow(event, throttle,  in perf_swevent_overflow()
7856 static void perf_swevent_event(struct perf_event *event, u64 nr,  in perf_swevent_event()  argument
7860 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_event()
7862 	local64_add(nr, &event->count);  in perf_swevent_event()
7867 	if (!is_sampling_event(event))  in perf_swevent_event()
7870 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {  in perf_swevent_event()
7872 		return perf_swevent_overflow(event, 1, data, regs);  in perf_swevent_event()
7874 		data->period = event->hw.last_period;  in perf_swevent_event()
7876 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)  in perf_swevent_event()
7877 		return perf_swevent_overflow(event, 1, data, regs);  in perf_swevent_event()
7882 	perf_swevent_overflow(event, 0, data, regs);  in perf_swevent_event()
7885 static int perf_exclude_event(struct perf_event *event,  in perf_exclude_event()  argument
7888 	if (event->hw.state & PERF_HES_STOPPED)  in perf_exclude_event()
7892 		if (event->attr.exclude_user && user_mode(regs))  in perf_exclude_event()
7895 		if (event->attr.exclude_kernel && !user_mode(regs))  in perf_exclude_event()
7902 static int perf_swevent_match(struct perf_event *event,  in perf_swevent_match()  argument
7908 	if (event->attr.type != type)  in perf_swevent_match()
7911 	if (event->attr.config != event_id)  in perf_swevent_match()
7914 	if (perf_exclude_event(event, regs))  in perf_swevent_match()
7950 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)  in find_swevent_head()  argument
7953 	u32 event_id = event->attr.config;  in find_swevent_head()
7954 	u64 type = event->attr.type;  in find_swevent_head()
7962 					  lockdep_is_held(&event->ctx->lock));  in find_swevent_head()
7975 	struct perf_event *event;  in do_perf_sw_event()  local
7983 	hlist_for_each_entry_rcu(event, head, hlist_entry) {  in do_perf_sw_event()
7984 		if (perf_swevent_match(event, type, event_id, data, regs))  in do_perf_sw_event()
7985 			perf_swevent_event(event, nr, data, regs);  in do_perf_sw_event()
8035 static void perf_swevent_read(struct perf_event *event)  in perf_swevent_read()  argument
8039 static int perf_swevent_add(struct perf_event *event, int flags)  in perf_swevent_add()  argument
8042 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_add()
8045 	if (is_sampling_event(event)) {  in perf_swevent_add()
8047 		perf_swevent_set_period(event);  in perf_swevent_add()
8052 	head = find_swevent_head(swhash, event);  in perf_swevent_add()
8056 	hlist_add_head_rcu(&event->hlist_entry, head);  in perf_swevent_add()
8057 	perf_event_update_userpage(event);  in perf_swevent_add()
8062 static void perf_swevent_del(struct perf_event *event, int flags)  in perf_swevent_del()  argument
8064 	hlist_del_rcu(&event->hlist_entry);  in perf_swevent_del()
8067 static void perf_swevent_start(struct perf_event *event, int flags)  in perf_swevent_start()  argument
8069 	event->hw.state = 0;  in perf_swevent_start()
8072 static void perf_swevent_stop(struct perf_event *event, int flags)  in perf_swevent_stop()  argument
8074 	event->hw.state = PERF_HES_STOPPED;  in perf_swevent_stop()
8166 static void sw_perf_event_destroy(struct perf_event *event)  in sw_perf_event_destroy()  argument
8168 	u64 event_id = event->attr.config;  in sw_perf_event_destroy()
8170 	WARN_ON(event->parent);  in sw_perf_event_destroy()
8176 static int perf_swevent_init(struct perf_event *event)  in perf_swevent_init()  argument
8178 	u64 event_id = event->attr.config;  in perf_swevent_init()
8180 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in perf_swevent_init()
8186 	if (has_branch_stack(event))  in perf_swevent_init()
8201 	if (!event->parent) {  in perf_swevent_init()
8209 		event->destroy = sw_perf_event_destroy;  in perf_swevent_init()
8230 static int perf_tp_filter_match(struct perf_event *event,  in perf_tp_filter_match()  argument
8236 	if (event->parent)  in perf_tp_filter_match()
8237 		event = event->parent;  in perf_tp_filter_match()
8239 	if (likely(!event->filter) || filter_match_preds(event->filter, record))  in perf_tp_filter_match()
8244 static int perf_tp_event_match(struct perf_event *event,  in perf_tp_event_match()  argument
8248 	if (event->hw.state & PERF_HES_STOPPED)  in perf_tp_event_match()
8253 	if (event->attr.exclude_kernel)  in perf_tp_event_match()
8256 	if (!perf_tp_filter_match(event, data))  in perf_tp_event_match()
8274 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,  in perf_trace_run_bpf_submit()
8284 	struct perf_event *event;  in perf_tp_event()  local
8298 	hlist_for_each_entry_rcu(event, head, hlist_entry) {  in perf_tp_event()
8299 		if (perf_tp_event_match(event, &data, regs))  in perf_tp_event()
8300 			perf_swevent_event(event, count, &data, regs);  in perf_tp_event()
8316 		list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_tp_event()
8317 			if (event->cpu != smp_processor_id())  in perf_tp_event()
8319 			if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event()
8321 			if (event->attr.config != entry->type)  in perf_tp_event()
8323 			if (perf_tp_event_match(event, &data, regs))  in perf_tp_event()
8324 				perf_swevent_event(event, count, &data, regs);  in perf_tp_event()
8334 static void tp_perf_event_destroy(struct perf_event *event)  in tp_perf_event_destroy()  argument
8336 	perf_trace_destroy(event);  in tp_perf_event_destroy()
8339 static int perf_tp_event_init(struct perf_event *event)  in perf_tp_event_init()  argument
8343 	if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event_init()
8349 	if (has_branch_stack(event))  in perf_tp_event_init()
8352 	err = perf_trace_init(event);  in perf_tp_event_init()
8356 	event->destroy = tp_perf_event_destroy;  in perf_tp_event_init()
8403 static int perf_kprobe_event_init(struct perf_event *event);
8415 static int perf_kprobe_event_init(struct perf_event *event)  in perf_kprobe_event_init()  argument
8420 	if (event->attr.type != perf_kprobe.type)  in perf_kprobe_event_init()
8429 	if (has_branch_stack(event))  in perf_kprobe_event_init()
8432 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_kprobe_event_init()
8433 	err = perf_kprobe_init(event, is_retprobe);  in perf_kprobe_event_init()
8437 	event->destroy = perf_kprobe_destroy;  in perf_kprobe_event_init()
8444 static int perf_uprobe_event_init(struct perf_event *event);
8456 static int perf_uprobe_event_init(struct perf_event *event)  in perf_uprobe_event_init()  argument
8461 	if (event->attr.type != perf_uprobe.type)  in perf_uprobe_event_init()
8470 	if (has_branch_stack(event))  in perf_uprobe_event_init()
8473 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_uprobe_event_init()
8474 	err = perf_uprobe_init(event, is_retprobe);  in perf_uprobe_event_init()
8478 	event->destroy = perf_uprobe_destroy;  in perf_uprobe_event_init()
8495 static void perf_event_free_filter(struct perf_event *event)  in perf_event_free_filter()  argument
8497 	ftrace_profile_free_filter(event);  in perf_event_free_filter()
8501 static void bpf_overflow_handler(struct perf_event *event,  in bpf_overflow_handler()  argument
8507 		.event = event,  in bpf_overflow_handler()
8516 	ret = BPF_PROG_RUN(event->prog, &ctx);  in bpf_overflow_handler()
8524 	event->orig_overflow_handler(event, data, regs);  in bpf_overflow_handler()
8527 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)  in perf_event_set_bpf_handler()  argument
8531 	if (event->overflow_handler_context)  in perf_event_set_bpf_handler()
8535 	if (event->prog)  in perf_event_set_bpf_handler()
8542 	event->prog = prog;  in perf_event_set_bpf_handler()
8543 	event->orig_overflow_handler = READ_ONCE(event->overflow_handler);  in perf_event_set_bpf_handler()
8544 	WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);  in perf_event_set_bpf_handler()
8548 static void perf_event_free_bpf_handler(struct perf_event *event)  in perf_event_free_bpf_handler()  argument
8550 	struct bpf_prog *prog = event->prog;  in perf_event_free_bpf_handler()
8555 	WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);  in perf_event_free_bpf_handler()
8556 	event->prog = NULL;  in perf_event_free_bpf_handler()
8560 static int perf_event_set_bpf_handler(struct perf_event *event, u32 prog_fd)  in perf_event_set_bpf_handler()  argument
8564 static void perf_event_free_bpf_handler(struct perf_event *event)  in perf_event_free_bpf_handler()  argument
8573 static inline bool perf_event_is_tracing(struct perf_event *event)  in perf_event_is_tracing()  argument
8575 	if (event->pmu == &perf_tracepoint)  in perf_event_is_tracing()
8578 	if (event->pmu == &perf_kprobe)  in perf_event_is_tracing()
8582 	if (event->pmu == &perf_uprobe)  in perf_event_is_tracing()
8588 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)  in perf_event_set_bpf_prog()  argument
8594 	if (!perf_event_is_tracing(event))  in perf_event_set_bpf_prog()
8595 		return perf_event_set_bpf_handler(event, prog_fd);  in perf_event_set_bpf_prog()
8597 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE;  in perf_event_set_bpf_prog()
8598 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;  in perf_event_set_bpf_prog()
8599 	is_syscall_tp = is_syscall_trace_event(event->tp_event);  in perf_event_set_bpf_prog()
8618 	    !(event->tp_event->flags & TRACE_EVENT_FL_KPROBE)) {  in perf_event_set_bpf_prog()
8624 		int off = trace_event_get_offsets(event->tp_event);  in perf_event_set_bpf_prog()
8632 	ret = perf_event_attach_bpf_prog(event, prog);  in perf_event_set_bpf_prog()
8638 static void perf_event_free_bpf_prog(struct perf_event *event)  in perf_event_free_bpf_prog()  argument
8640 	if (!perf_event_is_tracing(event)) {  in perf_event_free_bpf_prog()
8641 		perf_event_free_bpf_handler(event);  in perf_event_free_bpf_prog()
8644 	perf_event_detach_bpf_prog(event);  in perf_event_free_bpf_prog()
8653 static void perf_event_free_filter(struct perf_event *event)  in perf_event_free_filter()  argument
8657 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd)  in perf_event_set_bpf_prog()  argument
8662 static void perf_event_free_bpf_prog(struct perf_event *event)  in perf_event_free_bpf_prog()  argument
8684 perf_addr_filter_new(struct perf_event *event, struct list_head *filters)  in perf_addr_filter_new()  argument
8686 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);  in perf_addr_filter_new()
8713 static void perf_addr_filters_splice(struct perf_event *event,  in perf_addr_filters_splice()  argument
8719 	if (!has_addr_filter(event))  in perf_addr_filters_splice()
8723 	if (event->parent)  in perf_addr_filters_splice()
8726 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
8728 	list_splice_init(&event->addr_filters.list, &list);  in perf_addr_filters_splice()
8730 		list_splice(head, &event->addr_filters.list);  in perf_addr_filters_splice()
8732 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
8768 static void perf_event_addr_filters_apply(struct perf_event *event)  in perf_event_addr_filters_apply()  argument
8770 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_apply()
8771 	struct task_struct *task = READ_ONCE(event->ctx->task);  in perf_event_addr_filters_apply()
8787 	mm = get_task_mm(event->ctx->task);  in perf_event_addr_filters_apply()
8795 		event->addr_filters_offs[count] = 0;  in perf_event_addr_filters_apply()
8802 			event->addr_filters_offs[count] =  in perf_event_addr_filters_apply()
8808 	event->addr_filters_gen++;  in perf_event_addr_filters_apply()
8816 	perf_event_stop(event, 1);  in perf_event_addr_filters_apply()
8870 perf_event_parse_addr_filter(struct perf_event *event, char *fstr,  in perf_event_parse_addr_filter()  argument
8897 			filter = perf_addr_filter_new(event, filters);  in perf_event_parse_addr_filter()
8959 			if (kernel && event->attr.exclude_kernel)  in perf_event_parse_addr_filter()
8983 				if (!event->ctx->task)  in perf_event_parse_addr_filter()
9001 				event->addr_filters.nr_file_filters++;  in perf_event_parse_addr_filter()
9027 perf_event_set_addr_filter(struct perf_event *event, char *filter_str)  in perf_event_set_addr_filter()  argument
9036 	lockdep_assert_held(&event->ctx->mutex);  in perf_event_set_addr_filter()
9038 	if (WARN_ON_ONCE(event->parent))  in perf_event_set_addr_filter()
9041 	ret = perf_event_parse_addr_filter(event, filter_str, &filters);  in perf_event_set_addr_filter()
9045 	ret = event->pmu->addr_filters_validate(&filters);  in perf_event_set_addr_filter()
9050 	perf_addr_filters_splice(event, &filters);  in perf_event_set_addr_filter()
9053 	perf_event_for_each_child(event, perf_event_addr_filters_apply);  in perf_event_set_addr_filter()
9061 	event->addr_filters.nr_file_filters = 0;  in perf_event_set_addr_filter()
9066 static int perf_event_set_filter(struct perf_event *event, void __user *arg)  in perf_event_set_filter()  argument
9076 	if (perf_event_is_tracing(event)) {  in perf_event_set_filter()
9077 		struct perf_event_context *ctx = event->ctx;  in perf_event_set_filter()
9091 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);  in perf_event_set_filter()
9095 	if (has_addr_filter(event))  in perf_event_set_filter()
9096 		ret = perf_event_set_addr_filter(event, filter_str);  in perf_event_set_filter()
9111 	struct perf_event *event;  in perf_swevent_hrtimer()  local
9114 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);  in perf_swevent_hrtimer()
9116 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_swevent_hrtimer()
9119 	event->pmu->read(event);  in perf_swevent_hrtimer()
9121 	perf_sample_data_init(&data, 0, event->hw.last_period);  in perf_swevent_hrtimer()
9124 	if (regs && !perf_exclude_event(event, regs)) {  in perf_swevent_hrtimer()
9125 		if (!(event->attr.exclude_idle && is_idle_task(current)))  in perf_swevent_hrtimer()
9126 			if (__perf_event_overflow(event, 1, &data, regs))  in perf_swevent_hrtimer()
9130 	period = max_t(u64, 10000, event->hw.sample_period);  in perf_swevent_hrtimer()
9136 static void perf_swevent_start_hrtimer(struct perf_event *event)  in perf_swevent_start_hrtimer()  argument
9138 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_start_hrtimer()
9141 	if (!is_sampling_event(event))  in perf_swevent_start_hrtimer()
9157 static void perf_swevent_cancel_hrtimer(struct perf_event *event)  in perf_swevent_cancel_hrtimer()  argument
9159 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_cancel_hrtimer()
9161 	if (is_sampling_event(event)) {  in perf_swevent_cancel_hrtimer()
9169 static void perf_swevent_init_hrtimer(struct perf_event *event)  in perf_swevent_init_hrtimer()  argument
9171 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_init_hrtimer()
9173 	if (!is_sampling_event(event))  in perf_swevent_init_hrtimer()
9183 	if (event->attr.freq) {  in perf_swevent_init_hrtimer()
9184 		long freq = event->attr.sample_freq;  in perf_swevent_init_hrtimer()
9186 		event->attr.sample_period = NSEC_PER_SEC / freq;  in perf_swevent_init_hrtimer()
9187 		hwc->sample_period = event->attr.sample_period;  in perf_swevent_init_hrtimer()
9190 		event->attr.freq = 0;  in perf_swevent_init_hrtimer()
9198 static void cpu_clock_event_update(struct perf_event *event)  in cpu_clock_event_update()  argument
9204 	prev = local64_xchg(&event->hw.prev_count, now);  in cpu_clock_event_update()
9205 	local64_add(now - prev, &event->count);  in cpu_clock_event_update()
9208 static void cpu_clock_event_start(struct perf_event *event, int flags)  in cpu_clock_event_start()  argument
9210 	local64_set(&event->hw.prev_count, local_clock());  in cpu_clock_event_start()
9211 	perf_swevent_start_hrtimer(event);  in cpu_clock_event_start()
9214 static void cpu_clock_event_stop(struct perf_event *event, int flags)  in cpu_clock_event_stop()  argument
9216 	perf_swevent_cancel_hrtimer(event);  in cpu_clock_event_stop()
9217 	cpu_clock_event_update(event);  in cpu_clock_event_stop()
9220 static int cpu_clock_event_add(struct perf_event *event, int flags)  in cpu_clock_event_add()  argument
9223 		cpu_clock_event_start(event, flags);  in cpu_clock_event_add()
9224 	perf_event_update_userpage(event);  in cpu_clock_event_add()
9229 static void cpu_clock_event_del(struct perf_event *event, int flags)  in cpu_clock_event_del()  argument
9231 	cpu_clock_event_stop(event, flags);  in cpu_clock_event_del()
9234 static void cpu_clock_event_read(struct perf_event *event)  in cpu_clock_event_read()  argument
9236 	cpu_clock_event_update(event);  in cpu_clock_event_read()
9239 static int cpu_clock_event_init(struct perf_event *event)  in cpu_clock_event_init()  argument
9241 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in cpu_clock_event_init()
9244 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)  in cpu_clock_event_init()
9250 	if (has_branch_stack(event))  in cpu_clock_event_init()
9253 	perf_swevent_init_hrtimer(event);  in cpu_clock_event_init()
9275 static void task_clock_event_update(struct perf_event *event, u64 now)  in task_clock_event_update()  argument
9280 	prev = local64_xchg(&event->hw.prev_count, now);  in task_clock_event_update()
9282 	local64_add(delta, &event->count);  in task_clock_event_update()
9285 static void task_clock_event_start(struct perf_event *event, int flags)  in task_clock_event_start()  argument
9287 	local64_set(&event->hw.prev_count, event->ctx->time);  in task_clock_event_start()
9288 	perf_swevent_start_hrtimer(event);  in task_clock_event_start()
9291 static void task_clock_event_stop(struct perf_event *event, int flags)  in task_clock_event_stop()  argument
9293 	perf_swevent_cancel_hrtimer(event);  in task_clock_event_stop()
9294 	task_clock_event_update(event, event->ctx->time);  in task_clock_event_stop()
9297 static int task_clock_event_add(struct perf_event *event, int flags)  in task_clock_event_add()  argument
9300 		task_clock_event_start(event, flags);  in task_clock_event_add()
9301 	perf_event_update_userpage(event);  in task_clock_event_add()
9306 static void task_clock_event_del(struct perf_event *event, int flags)  in task_clock_event_del()  argument
9308 	task_clock_event_stop(event, PERF_EF_UPDATE);  in task_clock_event_del()
9311 static void task_clock_event_read(struct perf_event *event)  in task_clock_event_read()  argument
9314 	u64 delta = now - event->ctx->timestamp;  in task_clock_event_read()
9315 	u64 time = event->ctx->time + delta;  in task_clock_event_read()
9317 	task_clock_event_update(event, time);  in task_clock_event_read()
9320 static int task_clock_event_init(struct perf_event *event)  in task_clock_event_init()  argument
9322 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in task_clock_event_init()
9325 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)  in task_clock_event_init()
9331 	if (has_branch_stack(event))  in task_clock_event_init()
9334 	perf_swevent_init_hrtimer(event);  in task_clock_event_init()
9402 static int perf_event_idx_default(struct perf_event *event)  in perf_event_idx_default()  argument
9716 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)  in perf_try_init_event()  argument
9730 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {  in perf_try_init_event()
9735 		ctx = perf_event_ctx_lock_nested(event->group_leader,  in perf_try_init_event()
9740 	event->pmu = pmu;  in perf_try_init_event()
9741 	ret = pmu->event_init(event);  in perf_try_init_event()
9744 		perf_event_ctx_unlock(event->group_leader, ctx);  in perf_try_init_event()
9752 static struct pmu *perf_init_event(struct perf_event *event)  in perf_init_event()  argument
9761 	if (event->parent && event->parent->pmu) {  in perf_init_event()
9762 		pmu = event->parent->pmu;  in perf_init_event()
9763 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
9769 	pmu = idr_find(&pmu_idr, event->attr.type);  in perf_init_event()
9772 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
9779 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
9795 static void attach_sb_event(struct perf_event *event)  in attach_sb_event()  argument
9797 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in attach_sb_event()
9800 	list_add_rcu(&event->sb_list, &pel->list);  in attach_sb_event()
9811 static void account_pmu_sb_event(struct perf_event *event)  in account_pmu_sb_event()  argument
9813 	if (is_sb_event(event))  in account_pmu_sb_event()
9814 		attach_sb_event(event);  in account_pmu_sb_event()
9817 static void account_event_cpu(struct perf_event *event, int cpu)  in account_event_cpu()  argument
9819 	if (event->parent)  in account_event_cpu()
9822 	if (is_cgroup_event(event))  in account_event_cpu()
9847 static void account_event(struct perf_event *event)  in account_event()  argument
9851 	if (event->parent)  in account_event()
9854 	if (event->attach_state & PERF_ATTACH_TASK)  in account_event()
9856 	if (event->attr.mmap || event->attr.mmap_data)  in account_event()
9858 	if (event->attr.comm)  in account_event()
9860 	if (event->attr.namespaces)  in account_event()
9862 	if (event->attr.task)  in account_event()
9864 	if (event->attr.freq)  in account_event()
9866 	if (event->attr.context_switch) {  in account_event()
9870 	if (has_branch_stack(event))  in account_event()
9872 	if (is_cgroup_event(event))  in account_event()
9903 	account_event_cpu(event, event->cpu);  in account_event()
9905 	account_pmu_sb_event(event);  in account_event()
9920 	struct perf_event *event;  in perf_event_alloc()  local
9929 	event = kzalloc(sizeof(*event), GFP_KERNEL);  in perf_event_alloc()
9930 	if (!event)  in perf_event_alloc()
9938 		group_leader = event;  in perf_event_alloc()
9940 	mutex_init(&event->child_mutex);  in perf_event_alloc()
9941 	INIT_LIST_HEAD(&event->child_list);  in perf_event_alloc()
9943 	INIT_LIST_HEAD(&event->event_entry);  in perf_event_alloc()
9944 	INIT_LIST_HEAD(&event->sibling_list);  in perf_event_alloc()
9945 	INIT_LIST_HEAD(&event->active_list);  in perf_event_alloc()
9946 	init_event_group(event);  in perf_event_alloc()
9947 	INIT_LIST_HEAD(&event->rb_entry);  in perf_event_alloc()
9948 	INIT_LIST_HEAD(&event->active_entry);  in perf_event_alloc()
9949 	INIT_LIST_HEAD(&event->addr_filters.list);  in perf_event_alloc()
9950 	INIT_HLIST_NODE(&event->hlist_entry);  in perf_event_alloc()
9953 	init_waitqueue_head(&event->waitq);  in perf_event_alloc()
9954 	init_irq_work(&event->pending, perf_pending_event);  in perf_event_alloc()
9956 	mutex_init(&event->mmap_mutex);  in perf_event_alloc()
9957 	raw_spin_lock_init(&event->addr_filters.lock);  in perf_event_alloc()
9959 	atomic_long_set(&event->refcount, 1);  in perf_event_alloc()
9960 	event->cpu		= cpu;  in perf_event_alloc()
9961 	event->attr		= *attr;  in perf_event_alloc()
9962 	event->group_leader	= group_leader;  in perf_event_alloc()
9963 	event->pmu		= NULL;  in perf_event_alloc()
9964 	event->oncpu		= -1;  in perf_event_alloc()
9966 	event->parent		= parent_event;  in perf_event_alloc()
9968 	event->ns		= get_pid_ns(task_active_pid_ns(current));  in perf_event_alloc()
9969 	event->id		= atomic64_inc_return(&perf_event_id);  in perf_event_alloc()
9971 	event->state		= PERF_EVENT_STATE_INACTIVE;  in perf_event_alloc()
9974 		event->attach_state = PERF_ATTACH_TASK;  in perf_event_alloc()
9981 		event->hw.target = task;  in perf_event_alloc()
9984 	event->clock = &local_clock;  in perf_event_alloc()
9986 		event->clock = parent_event->clock;  in perf_event_alloc()
9999 			event->prog = prog;  in perf_event_alloc()
10000 			event->orig_overflow_handler =  in perf_event_alloc()
10007 		event->overflow_handler	= overflow_handler;  in perf_event_alloc()
10008 		event->overflow_handler_context = context;  in perf_event_alloc()
10009 	} else if (is_write_backward(event)){  in perf_event_alloc()
10010 		event->overflow_handler = perf_event_output_backward;  in perf_event_alloc()
10011 		event->overflow_handler_context = NULL;  in perf_event_alloc()
10013 		event->overflow_handler = perf_event_output_forward;  in perf_event_alloc()
10014 		event->overflow_handler_context = NULL;  in perf_event_alloc()
10017 	perf_event__state_init(event);  in perf_event_alloc()
10021 	hwc = &event->hw;  in perf_event_alloc()
10036 	if (!has_branch_stack(event))  in perf_event_alloc()
10037 		event->attr.branch_sample_type = 0;  in perf_event_alloc()
10040 		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);  in perf_event_alloc()
10045 	pmu = perf_init_event(event);  in perf_event_alloc()
10051 	err = exclusive_event_init(event);  in perf_event_alloc()
10055 	if (has_addr_filter(event)) {  in perf_event_alloc()
10056 		event->addr_filters_offs = kcalloc(pmu->nr_addr_filters,  in perf_event_alloc()
10059 		if (!event->addr_filters_offs) {  in perf_event_alloc()
10065 		event->addr_filters_gen = 1;  in perf_event_alloc()
10068 	if (!event->parent) {  in perf_event_alloc()
10069 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {  in perf_event_alloc()
10077 	account_event(event);  in perf_event_alloc()
10079 	return event;  in perf_event_alloc()
10082 	kfree(event->addr_filters_offs);  in perf_event_alloc()
10085 	exclusive_event_destroy(event);  in perf_event_alloc()
10088 	if (event->destroy)  in perf_event_alloc()
10089 		event->destroy(event);  in perf_event_alloc()
10092 	if (is_cgroup_event(event))  in perf_event_alloc()
10093 		perf_detach_cgroup(event);  in perf_event_alloc()
10094 	if (event->ns)  in perf_event_alloc()
10095 		put_pid_ns(event->ns);  in perf_event_alloc()
10096 	if (event->hw.target)  in perf_event_alloc()
10097 		put_task_struct(event->hw.target);  in perf_event_alloc()
10098 	kfree(event);  in perf_event_alloc()
10239 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)  in perf_event_set_output()  argument
10248 	if (event == output_event)  in perf_event_set_output()
10254 	if (output_event->cpu != event->cpu)  in perf_event_set_output()
10260 	if (output_event->cpu == -1 && output_event->ctx != event->ctx)  in perf_event_set_output()
10266 	if (output_event->clock != event->clock)  in perf_event_set_output()
10273 	if (is_write_backward(output_event) != is_write_backward(event))  in perf_event_set_output()
10279 	if (has_aux(event) && has_aux(output_event) &&  in perf_event_set_output()
10280 	    event->pmu != output_event->pmu)  in perf_event_set_output()
10284 	mutex_lock(&event->mmap_mutex);  in perf_event_set_output()
10286 	if (atomic_read(&event->mmap_count))  in perf_event_set_output()
10296 	ring_buffer_attach(event, rb);  in perf_event_set_output()
10300 	mutex_unlock(&event->mmap_mutex);  in perf_event_set_output()
10315 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)  in perf_event_set_clock()  argument
10321 		event->clock = &ktime_get_mono_fast_ns;  in perf_event_set_clock()
10326 		event->clock = &ktime_get_raw_fast_ns;  in perf_event_set_clock()
10331 		event->clock = &ktime_get_real_ns;  in perf_event_set_clock()
10335 		event->clock = &ktime_get_boot_ns;  in perf_event_set_clock()
10339 		event->clock = &ktime_get_tai_ns;  in perf_event_set_clock()
10346 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))  in perf_event_set_clock()
10396 	struct perf_event *event, *sibling;  in SYSCALL_DEFINE5()  local
10502 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,  in SYSCALL_DEFINE5()
10504 	if (IS_ERR(event)) {  in SYSCALL_DEFINE5()
10505 		err = PTR_ERR(event);  in SYSCALL_DEFINE5()
10509 	if (is_sampling_event(event)) {  in SYSCALL_DEFINE5()
10510 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {  in SYSCALL_DEFINE5()
10520 	pmu = event->pmu;  in SYSCALL_DEFINE5()
10523 		err = perf_event_set_clock(event, attr.clockid);  in SYSCALL_DEFINE5()
10529 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in SYSCALL_DEFINE5()
10532 		if (is_software_event(event) &&  in SYSCALL_DEFINE5()
10543 		} else if (!is_software_event(event) &&  in SYSCALL_DEFINE5()
10558 	ctx = find_get_context(pmu, task, event);  in SYSCALL_DEFINE5()
10583 		if (group_leader->clock != event->clock)  in SYSCALL_DEFINE5()
10591 		if (group_leader->cpu != event->cpu)  in SYSCALL_DEFINE5()
10617 		err = perf_event_set_output(event, output_event);  in SYSCALL_DEFINE5()
10622 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,  in SYSCALL_DEFINE5()
10665 	if (!perf_event_validate_size(event)) {  in SYSCALL_DEFINE5()
10691 	if (!exclusive_event_installable(event, ctx)) {  in SYSCALL_DEFINE5()
10757 	perf_event__header_size(event);  in SYSCALL_DEFINE5()
10758 	perf_event__id_header_size(event);  in SYSCALL_DEFINE5()
10760 	event->owner = current;  in SYSCALL_DEFINE5()
10762 	perf_install_in_context(ctx, event, event->cpu);  in SYSCALL_DEFINE5()
10775 	list_add_tail(&event->owner_entry, &current->perf_event_list);  in SYSCALL_DEFINE5()
10803 		free_event(event);  in SYSCALL_DEFINE5()
10831 	struct perf_event *event;  in perf_event_create_kernel_counter()  local
10838 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,  in perf_event_create_kernel_counter()
10840 	if (IS_ERR(event)) {  in perf_event_create_kernel_counter()
10841 		err = PTR_ERR(event);  in perf_event_create_kernel_counter()
10846 	event->owner = TASK_TOMBSTONE;  in perf_event_create_kernel_counter()
10848 	ctx = find_get_context(event->pmu, task, event);  in perf_event_create_kernel_counter()
10876 	if (!exclusive_event_installable(event, ctx)) {  in perf_event_create_kernel_counter()
10881 	perf_install_in_context(ctx, event, cpu);  in perf_event_create_kernel_counter()
10885 	return event;  in perf_event_create_kernel_counter()
10892 	free_event(event);  in perf_event_create_kernel_counter()
10902 	struct perf_event *event, *tmp;  in perf_pmu_migrate_context()  local
10913 	list_for_each_entry_safe(event, tmp, &src_ctx->event_list,  in perf_pmu_migrate_context()
10915 		perf_remove_from_context(event, 0);  in perf_pmu_migrate_context()
10916 		unaccount_event_cpu(event, src_cpu);  in perf_pmu_migrate_context()
10918 		list_add(&event->migrate_entry, &events);  in perf_pmu_migrate_context()
10934 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {  in perf_pmu_migrate_context()
10935 		if (event->group_leader == event)  in perf_pmu_migrate_context()
10938 		list_del(&event->migrate_entry);  in perf_pmu_migrate_context()
10939 		if (event->state >= PERF_EVENT_STATE_OFF)  in perf_pmu_migrate_context()
10940 			event->state = PERF_EVENT_STATE_INACTIVE;  in perf_pmu_migrate_context()
10941 		account_event_cpu(event, dst_cpu);  in perf_pmu_migrate_context()
10942 		perf_install_in_context(dst_ctx, event, dst_cpu);  in perf_pmu_migrate_context()
10950 	list_for_each_entry_safe(event, tmp, &events, migrate_entry) {  in perf_pmu_migrate_context()
10951 		list_del(&event->migrate_entry);  in perf_pmu_migrate_context()
10952 		if (event->state >= PERF_EVENT_STATE_OFF)  in perf_pmu_migrate_context()
10953 			event->state = PERF_EVENT_STATE_INACTIVE;  in perf_pmu_migrate_context()
10954 		account_event_cpu(event, dst_cpu);  in perf_pmu_migrate_context()
10955 		perf_install_in_context(dst_ctx, event, dst_cpu);  in perf_pmu_migrate_context()
11110 	struct perf_event *event, *tmp;  in perf_event_exit_task()  local
11114 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,  in perf_event_exit_task()
11116 		list_del_init(&event->owner_entry);  in perf_event_exit_task()
11123 		smp_store_release(&event->owner, NULL);  in perf_event_exit_task()
11139 static void perf_free_event(struct perf_event *event,  in perf_free_event()  argument
11142 	struct perf_event *parent = event->parent;  in perf_free_event()
11148 	list_del_init(&event->child_list);  in perf_free_event()
11154 	perf_group_detach(event);  in perf_free_event()
11155 	list_del_event(event, ctx);  in perf_free_event()
11157 	free_event(event);  in perf_free_event()
11170 	struct perf_event *event, *tmp;  in perf_event_free_task()  local
11191 		list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)  in perf_event_free_task()
11192 			perf_free_event(event, ctx);  in perf_event_free_task()
11231 const struct perf_event_attr *perf_event_attrs(struct perf_event *event)  in perf_event_attrs()  argument
11233 	if (!event)  in perf_event_attrs()
11236 	return &event->attr;  in perf_event_attrs()
11403 inherit_task_group(struct perf_event *event, struct task_struct *parent,  in inherit_task_group()  argument
11411 	if (!event->attr.inherit) {  in inherit_task_group()
11431 	ret = inherit_group(event, parent, parent_ctx,  in inherit_task_group()
11447 	struct perf_event *event;  in perf_event_init_context()  local
11481 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {  in perf_event_init_context()
11482 		ret = inherit_task_group(event, parent, parent_ctx,  in perf_event_init_context()
11497 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {  in perf_event_init_context()
11498 		ret = inherit_task_group(event, parent, parent_ctx,  in perf_event_init_context()
11602 	struct perf_event *event;  in __perf_event_exit_context()  local
11606 	list_for_each_entry(event, &ctx->event_list, event_entry)  in __perf_event_exit_context()
11607 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);  in __perf_event_exit_context()