176 static bool is_kernel_event(struct perf_event *event)  in is_kernel_event()  argument
178 	return READ_ONCE(event->owner) == TASK_TOMBSTONE;  in is_kernel_event()
212 	struct perf_event *event;  member
220 	struct perf_event *event = efs->event;  in event_function()  local
221 	struct perf_event_context *ctx = event->ctx;  in event_function()
256 	efs->func(event, cpuctx, ctx, efs->data);  in event_function()
263 static void event_function_call(struct perf_event *event, event_f func, void *data)  in event_function_call()  argument
265 	struct perf_event_context *ctx = event->ctx;  in event_function_call()
268 		.event = event,  in event_function_call()
273 	if (!event->parent) {  in event_function_call()
283 		cpu_function_call(event->cpu, event_function, &efs);  in event_function_call()
308 	func(event, NULL, ctx, data);  in event_function_call()
316 static void event_function_local(struct perf_event *event, event_f func, void *data)  in event_function_local()  argument
318 	struct perf_event_context *ctx = event->ctx;  in event_function_local()
355 	func(event, cpuctx, ctx, data);  in event_function_local()
574 static u64 perf_event_time(struct perf_event *event);
583 static inline u64 perf_event_clock(struct perf_event *event)  in perf_event_clock()  argument
585 	return event->clock();  in perf_event_clock()
611 __perf_effective_state(struct perf_event *event)  in __perf_effective_state()  argument
613 	struct perf_event *leader = event->group_leader;  in __perf_effective_state()
618 	return event->state;  in __perf_effective_state()
622 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running)  in __perf_update_times()  argument
624 	enum perf_event_state state = __perf_effective_state(event);  in __perf_update_times()
625 	u64 delta = now - event->tstamp;  in __perf_update_times()
627 	*enabled = event->total_time_enabled;  in __perf_update_times()
631 	*running = event->total_time_running;  in __perf_update_times()
636 static void perf_event_update_time(struct perf_event *event)  in perf_event_update_time()  argument
638 	u64 now = perf_event_time(event);  in perf_event_update_time()
640 	__perf_update_times(event, now, &event->total_time_enabled,  in perf_event_update_time()
641 					&event->total_time_running);  in perf_event_update_time()
642 	event->tstamp = now;  in perf_event_update_time()
654 perf_event_set_state(struct perf_event *event, enum perf_event_state state)  in perf_event_set_state()  argument
656 	if (event->state == state)  in perf_event_set_state()
659 	perf_event_update_time(event);  in perf_event_set_state()
664 	if ((event->state < 0) ^ (state < 0))  in perf_event_set_state()
665 		perf_event_update_sibling_time(event);  in perf_event_set_state()
667 	WRITE_ONCE(event->state, state);  in perf_event_set_state()
709 perf_cgroup_match(struct perf_event *event)  in perf_cgroup_match()  argument
714 	if (!event->cgrp)  in perf_cgroup_match()
728 				    event->cgrp->css.cgroup);  in perf_cgroup_match()
731 static inline void perf_detach_cgroup(struct perf_event *event)  in perf_detach_cgroup()  argument
733 	css_put(&event->cgrp->css);  in perf_detach_cgroup()
734 	event->cgrp = NULL;  in perf_detach_cgroup()
737 static inline int is_cgroup_event(struct perf_event *event)  in is_cgroup_event()  argument
739 	return event->cgrp != NULL;  in is_cgroup_event()
742 static inline u64 perf_cgroup_event_time(struct perf_event *event)  in perf_cgroup_event_time()  argument
746 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time()
750 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)  in perf_cgroup_event_time_now()  argument
754 	t = per_cpu_ptr(event->cgrp->info, event->cpu);  in perf_cgroup_event_time_now()
792 static inline void update_cgrp_time_from_event(struct perf_event *event)  in update_cgrp_time_from_event()  argument
800 	if (!is_cgroup_event(event))  in update_cgrp_time_from_event()
803 	info = this_cpu_ptr(event->cgrp->info);  in update_cgrp_time_from_event()
879 static int perf_cgroup_ensure_storage(struct perf_event *event,  in perf_cgroup_ensure_storage()  argument
920 static inline int perf_cgroup_connect(int fd, struct perf_event *event,  in perf_cgroup_connect()  argument
939 	ret = perf_cgroup_ensure_storage(event, css);  in perf_cgroup_connect()
944 	event->cgrp = cgrp;  in perf_cgroup_connect()
952 		perf_detach_cgroup(event);  in perf_cgroup_connect()
961 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)  in perf_cgroup_event_enable()  argument
965 	if (!is_cgroup_event(event))  in perf_cgroup_event_enable()
981 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)  in perf_cgroup_event_disable()  argument
985 	if (!is_cgroup_event(event))  in perf_cgroup_event_disable()
1003 perf_cgroup_match(struct perf_event *event)  in perf_cgroup_match()  argument
1008 static inline void perf_detach_cgroup(struct perf_event *event)  in perf_detach_cgroup()  argument
1011 static inline int is_cgroup_event(struct perf_event *event)  in is_cgroup_event()  argument
1016 static inline void update_cgrp_time_from_event(struct perf_event *event)  in update_cgrp_time_from_event()  argument
1025 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,  in perf_cgroup_connect()  argument
1037 static inline u64 perf_cgroup_event_time(struct perf_event *event)  in perf_cgroup_event_time()  argument
1042 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now)  in perf_cgroup_event_time_now()  argument
1048 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx)  in perf_cgroup_event_enable()  argument
1053 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx)  in perf_cgroup_event_disable()  argument
1256 perf_event_ctx_lock_nested(struct perf_event *event, int nesting)  in perf_event_ctx_lock_nested()  argument
1262 	ctx = READ_ONCE(event->ctx);  in perf_event_ctx_lock_nested()
1270 	if (event->ctx != ctx) {  in perf_event_ctx_lock_nested()
1280 perf_event_ctx_lock(struct perf_event *event)  in perf_event_ctx_lock()  argument
1282 	return perf_event_ctx_lock_nested(event, 0);  in perf_event_ctx_lock()
1285 static void perf_event_ctx_unlock(struct perf_event *event,  in perf_event_ctx_unlock()  argument
1311 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p,  in perf_event_pid_type()  argument
1318 	if (event->parent)  in perf_event_pid_type()
1319 		event = event->parent;  in perf_event_pid_type()
1321 	nr = __task_pid_nr_ns(p, type, event->ns);  in perf_event_pid_type()
1328 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)  in perf_event_pid()  argument
1330 	return perf_event_pid_type(event, p, PIDTYPE_TGID);  in perf_event_pid()
1333 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)  in perf_event_tid()  argument
1335 	return perf_event_pid_type(event, p, PIDTYPE_PID);  in perf_event_tid()
1342 static u64 primary_event_id(struct perf_event *event)  in primary_event_id()  argument
1344 	u64 id = event->id;  in primary_event_id()
1346 	if (event->parent)  in primary_event_id()
1347 		id = event->parent->id;  in primary_event_id()
1467 static u64 perf_event_time(struct perf_event *event)  in perf_event_time()  argument
1469 	struct perf_event_context *ctx = event->ctx;  in perf_event_time()
1474 	if (is_cgroup_event(event))  in perf_event_time()
1475 		return perf_cgroup_event_time(event);  in perf_event_time()
1480 static u64 perf_event_time_now(struct perf_event *event, u64 now)  in perf_event_time_now()  argument
1482 	struct perf_event_context *ctx = event->ctx;  in perf_event_time_now()
1487 	if (is_cgroup_event(event))  in perf_event_time_now()
1488 		return perf_cgroup_event_time_now(event, now);  in perf_event_time_now()
1497 static enum event_type_t get_event_type(struct perf_event *event)  in get_event_type()  argument
1499 	struct perf_event_context *ctx = event->ctx;  in get_event_type()
1508 	if (event->group_leader != event)  in get_event_type()
1509 		event = event->group_leader;  in get_event_type()
1511 	event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE;  in get_event_type()
1521 static void init_event_group(struct perf_event *event)  in init_event_group()  argument
1523 	RB_CLEAR_NODE(&event->group_node);  in init_event_group()
1524 	event->group_index = 0;  in init_event_group()
1532 get_event_groups(struct perf_event *event, struct perf_event_context *ctx)  in get_event_groups()  argument
1534 	if (event->attr.pinned)  in get_event_groups()
1549 static inline struct cgroup *event_cgroup(const struct perf_event *event)  in event_cgroup()  argument
1554 	if (event->cgrp)  in event_cgroup()
1555 		cgroup = event->cgrp->css.cgroup;  in event_cgroup()
1663 			 struct perf_event *event)  in perf_event_groups_insert()  argument
1665 	event->group_index = ++groups->index;  in perf_event_groups_insert()
1667 	rb_add(&event->group_node, &groups->tree, __group_less);  in perf_event_groups_insert()
1674 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx)  in add_event_to_groups()  argument
1678 	groups = get_event_groups(event, ctx);  in add_event_to_groups()
1679 	perf_event_groups_insert(groups, event);  in add_event_to_groups()
1687 			 struct perf_event *event)  in perf_event_groups_delete()  argument
1689 	WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) ||  in perf_event_groups_delete()
1692 	rb_erase(&event->group_node, &groups->tree);  in perf_event_groups_delete()
1693 	init_event_group(event);  in perf_event_groups_delete()
1700 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx)  in del_event_from_groups()  argument
1704 	groups = get_event_groups(event, ctx);  in del_event_from_groups()
1705 	perf_event_groups_delete(groups, event);  in del_event_from_groups()
1730 perf_event_groups_next(struct perf_event *event, struct pmu *pmu)  in perf_event_groups_next()  argument
1733 		.cpu = event->cpu,  in perf_event_groups_next()
1735 		.cgroup = event_cgroup(event),  in perf_event_groups_next()
1739 	next = rb_next_match(&key, &event->group_node, __group_cmp);  in perf_event_groups_next()
1746 #define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu)		\  argument
1747 	for (event = perf_event_groups_first(groups, cpu, pmu, NULL);	\
1748 	     event; event = perf_event_groups_next(event, pmu))
1753 #define perf_event_groups_for_each(event, groups)			\  argument
1754 	for (event = rb_entry_safe(rb_first(&((groups)->tree)),		\
1755 				typeof(*event), group_node); event;	\
1756 		event = rb_entry_safe(rb_next(&event->group_node),	\
1757 				typeof(*event), group_node))
1764 list_add_event(struct perf_event *event, struct perf_event_context *ctx)  in list_add_event()  argument
1768 	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);  in list_add_event()
1769 	event->attach_state |= PERF_ATTACH_CONTEXT;  in list_add_event()
1771 	event->tstamp = perf_event_time(event);  in list_add_event()
1778 	if (event->group_leader == event) {  in list_add_event()
1779 		event->group_caps = event->event_caps;  in list_add_event()
1780 		add_event_to_groups(event, ctx);  in list_add_event()
1783 	list_add_rcu(&event->event_entry, &ctx->event_list);  in list_add_event()
1785 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_add_event()
1787 	if (event->attr.inherit_stat)  in list_add_event()
1790 	if (event->state > PERF_EVENT_STATE_OFF)  in list_add_event()
1791 		perf_cgroup_event_enable(event, ctx);  in list_add_event()
1794 	event->pmu_ctx->nr_events++;  in list_add_event()
1800 static inline void perf_event__state_init(struct perf_event *event)  in perf_event__state_init()  argument
1802 	event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :  in perf_event__state_init()
1806 static void __perf_event_read_size(struct perf_event *event, int nr_siblings)  in __perf_event_read_size()  argument
1812 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)  in __perf_event_read_size()
1815 	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)  in __perf_event_read_size()
1818 	if (event->attr.read_format & PERF_FORMAT_ID)  in __perf_event_read_size()
1821 	if (event->attr.read_format & PERF_FORMAT_LOST)  in __perf_event_read_size()
1824 	if (event->attr.read_format & PERF_FORMAT_GROUP) {  in __perf_event_read_size()
1830 	event->read_size = size;  in __perf_event_read_size()
1833 static void __perf_event_header_size(struct perf_event *event, u64 sample_type)  in __perf_event_header_size()  argument
1851 		size += event->read_size;  in __perf_event_header_size()
1871 	event->header_size = size;  in __perf_event_header_size()
1878 static void perf_event__header_size(struct perf_event *event)  in perf_event__header_size()  argument
1880 	__perf_event_read_size(event,  in perf_event__header_size()
1881 			       event->group_leader->nr_siblings);  in perf_event__header_size()
1882 	__perf_event_header_size(event, event->attr.sample_type);  in perf_event__header_size()
1885 static void perf_event__id_header_size(struct perf_event *event)  in perf_event__id_header_size()  argument
1888 	u64 sample_type = event->attr.sample_type;  in perf_event__id_header_size()
1909 	event->id_header_size = size;  in perf_event__id_header_size()
1912 static bool perf_event_validate_size(struct perf_event *event)  in perf_event_validate_size()  argument
1918 	__perf_event_read_size(event, event->group_leader->nr_siblings + 1);  in perf_event_validate_size()
1919 	__perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ);  in perf_event_validate_size()
1920 	perf_event__id_header_size(event);  in perf_event_validate_size()
1926 	if (event->read_size + event->header_size +  in perf_event_validate_size()
1927 	    event->id_header_size + sizeof(struct perf_event_header) >= 16*1024)  in perf_event_validate_size()
1933 static void perf_group_attach(struct perf_event *event)  in perf_group_attach()  argument
1935 	struct perf_event *group_leader = event->group_leader, *pos;  in perf_group_attach()
1937 	lockdep_assert_held(&event->ctx->lock);  in perf_group_attach()
1943 	if (event->attach_state & PERF_ATTACH_GROUP)  in perf_group_attach()
1946 	event->attach_state |= PERF_ATTACH_GROUP;  in perf_group_attach()
1948 	if (group_leader == event)  in perf_group_attach()
1951 	WARN_ON_ONCE(group_leader->ctx != event->ctx);  in perf_group_attach()
1953 	group_leader->group_caps &= event->event_caps;  in perf_group_attach()
1955 	list_add_tail(&event->sibling_list, &group_leader->sibling_list);  in perf_group_attach()
1970 list_del_event(struct perf_event *event, struct perf_event_context *ctx)  in list_del_event()  argument
1972 	WARN_ON_ONCE(event->ctx != ctx);  in list_del_event()
1978 	if (!(event->attach_state & PERF_ATTACH_CONTEXT))  in list_del_event()
1981 	event->attach_state &= ~PERF_ATTACH_CONTEXT;  in list_del_event()
1984 	if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)  in list_del_event()
1986 	if (event->attr.inherit_stat)  in list_del_event()
1989 	list_del_rcu(&event->event_entry);  in list_del_event()
1991 	if (event->group_leader == event)  in list_del_event()
1992 		del_event_from_groups(event, ctx);  in list_del_event()
2001 	if (event->state > PERF_EVENT_STATE_OFF) {  in list_del_event()
2002 		perf_cgroup_event_disable(event, ctx);  in list_del_event()
2003 		perf_event_set_state(event, PERF_EVENT_STATE_OFF);  in list_del_event()
2007 	event->pmu_ctx->nr_events--;  in list_del_event()
2011 perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event)  in perf_aux_output_match()  argument
2016 	if (!event->pmu->aux_output_match)  in perf_aux_output_match()
2019 	return event->pmu->aux_output_match(aux_event);  in perf_aux_output_match()
2022 static void put_event(struct perf_event *event);
2023 static void event_sched_out(struct perf_event *event,
2026 static void perf_put_aux_event(struct perf_event *event)  in perf_put_aux_event()  argument
2028 	struct perf_event_context *ctx = event->ctx;  in perf_put_aux_event()
2034 	if (event->aux_event) {  in perf_put_aux_event()
2035 		iter = event->aux_event;  in perf_put_aux_event()
2036 		event->aux_event = NULL;  in perf_put_aux_event()
2045 	for_each_sibling_event(iter, event->group_leader) {  in perf_put_aux_event()
2046 		if (iter->aux_event != event)  in perf_put_aux_event()
2050 		put_event(event);  in perf_put_aux_event()
2058 		perf_event_set_state(event, PERF_EVENT_STATE_ERROR);  in perf_put_aux_event()
2062 static bool perf_need_aux_event(struct perf_event *event)  in perf_need_aux_event()  argument
2064 	return !!event->attr.aux_output || !!event->attr.aux_sample_size;  in perf_need_aux_event()
2067 static int perf_get_aux_event(struct perf_event *event,  in perf_get_aux_event()  argument
2082 	if (event->attr.aux_output && event->attr.aux_sample_size)  in perf_get_aux_event()
2085 	if (event->attr.aux_output &&  in perf_get_aux_event()
2086 	    !perf_aux_output_match(event, group_leader))  in perf_get_aux_event()
2089 	if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux)  in perf_get_aux_event()
2101 	event->aux_event = group_leader;  in perf_get_aux_event()
2106 static inline struct list_head *get_event_list(struct perf_event *event)  in get_event_list()  argument
2108 	return event->attr.pinned ? &event->pmu_ctx->pinned_active :  in get_event_list()
2109 				    &event->pmu_ctx->flexible_active;  in get_event_list()
2118 static inline void perf_remove_sibling_event(struct perf_event *event)  in perf_remove_sibling_event()  argument
2120 	event_sched_out(event, event->ctx);  in perf_remove_sibling_event()
2121 	perf_event_set_state(event, PERF_EVENT_STATE_ERROR);  in perf_remove_sibling_event()
2124 static void perf_group_detach(struct perf_event *event)  in perf_group_detach()  argument
2126 	struct perf_event *leader = event->group_leader;  in perf_group_detach()
2128 	struct perf_event_context *ctx = event->ctx;  in perf_group_detach()
2135 	if (!(event->attach_state & PERF_ATTACH_GROUP))  in perf_group_detach()
2138 	event->attach_state &= ~PERF_ATTACH_GROUP;  in perf_group_detach()
2140 	perf_put_aux_event(event);  in perf_group_detach()
2145 	if (leader != event) {  in perf_group_detach()
2146 		list_del_init(&event->sibling_list);  in perf_group_detach()
2147 		event->group_leader->nr_siblings--;  in perf_group_detach()
2148 		event->group_leader->group_generation++;  in perf_group_detach()
2157 	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) {  in perf_group_detach()
2166 		sibling->group_caps = event->group_caps;  in perf_group_detach()
2169 			add_event_to_groups(sibling, event->ctx);  in perf_group_detach()
2175 		WARN_ON_ONCE(sibling->ctx != event->ctx);  in perf_group_detach()
2187 static void perf_child_detach(struct perf_event *event)  in perf_child_detach()  argument
2189 	struct perf_event *parent_event = event->parent;  in perf_child_detach()
2191 	if (!(event->attach_state & PERF_ATTACH_CHILD))  in perf_child_detach()
2194 	event->attach_state &= ~PERF_ATTACH_CHILD;  in perf_child_detach()
2201 	sync_child_event(event);  in perf_child_detach()
2202 	list_del_init(&event->child_list);  in perf_child_detach()
2205 static bool is_orphaned_event(struct perf_event *event)  in is_orphaned_event()  argument
2207 	return event->state == PERF_EVENT_STATE_DEAD;  in is_orphaned_event()
2211 event_filter_match(struct perf_event *event)  in event_filter_match()  argument
2213 	return (event->cpu == -1 || event->cpu == smp_processor_id()) &&  in event_filter_match()
2214 	       perf_cgroup_match(event);  in event_filter_match()
2218 event_sched_out(struct perf_event *event, struct perf_event_context *ctx)  in event_sched_out()  argument
2220 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_out()
2226 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_out()
2229 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in event_sched_out()
2237 	list_del_init(&event->active_list);  in event_sched_out()
2239 	perf_pmu_disable(event->pmu);  in event_sched_out()
2241 	event->pmu->del(event, 0);  in event_sched_out()
2242 	event->oncpu = -1;  in event_sched_out()
2244 	if (event->pending_disable) {  in event_sched_out()
2245 		event->pending_disable = 0;  in event_sched_out()
2246 		perf_cgroup_event_disable(event, ctx);  in event_sched_out()
2250 	if (event->pending_sigtrap) {  in event_sched_out()
2253 		event->pending_sigtrap = 0;  in event_sched_out()
2255 		    !event->pending_work) {  in event_sched_out()
2256 			event->pending_work = 1;  in event_sched_out()
2258 			WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));  in event_sched_out()
2259 			task_work_add(current, &event->pending_task, TWA_RESUME);  in event_sched_out()
2262 			local_dec(&event->ctx->nr_pending);  in event_sched_out()
2265 	perf_event_set_state(event, state);  in event_sched_out()
2267 	if (!is_software_event(event))  in event_sched_out()
2269 	if (event->attr.freq && event->attr.sample_freq)  in event_sched_out()
2271 	if (event->attr.exclusive || !cpc->active_oncpu)  in event_sched_out()
2274 	perf_pmu_enable(event->pmu);  in event_sched_out()
2280 	struct perf_event *event;  in group_sched_out()  local
2292 	for_each_sibling_event(event, group_event)  in group_sched_out()
2293 		event_sched_out(event, ctx);  in group_sched_out()
2307 __perf_remove_from_context(struct perf_event *event,  in __perf_remove_from_context()  argument
2312 	struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx;  in __perf_remove_from_context()
2325 		event->pending_disable = 1;  in __perf_remove_from_context()
2326 	event_sched_out(event, ctx);  in __perf_remove_from_context()
2328 		perf_group_detach(event);  in __perf_remove_from_context()
2330 		perf_child_detach(event);  in __perf_remove_from_context()
2331 	list_del_event(event, ctx);  in __perf_remove_from_context()
2333 		event->state = PERF_EVENT_STATE_DEAD;  in __perf_remove_from_context()
2369 static void perf_remove_from_context(struct perf_event *event, unsigned long flags)  in perf_remove_from_context()  argument
2371 	struct perf_event_context *ctx = event->ctx;  in perf_remove_from_context()
2382 		__perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context),  in perf_remove_from_context()
2389 	event_function_call(event, __perf_remove_from_context, (void *)flags);  in perf_remove_from_context()
2395 static void __perf_event_disable(struct perf_event *event,  in __perf_event_disable()  argument
2400 	if (event->state < PERF_EVENT_STATE_INACTIVE)  in __perf_event_disable()
2405 		update_cgrp_time_from_event(event);  in __perf_event_disable()
2408 	perf_pmu_disable(event->pmu_ctx->pmu);  in __perf_event_disable()
2410 	if (event == event->group_leader)  in __perf_event_disable()
2411 		group_sched_out(event, ctx);  in __perf_event_disable()
2413 		event_sched_out(event, ctx);  in __perf_event_disable()
2415 	perf_event_set_state(event, PERF_EVENT_STATE_OFF);  in __perf_event_disable()
2416 	perf_cgroup_event_disable(event, ctx);  in __perf_event_disable()
2418 	perf_pmu_enable(event->pmu_ctx->pmu);  in __perf_event_disable()
2435 static void _perf_event_disable(struct perf_event *event)  in _perf_event_disable()  argument
2437 	struct perf_event_context *ctx = event->ctx;  in _perf_event_disable()
2440 	if (event->state <= PERF_EVENT_STATE_OFF) {  in _perf_event_disable()
2446 	event_function_call(event, __perf_event_disable, NULL);  in _perf_event_disable()
2449 void perf_event_disable_local(struct perf_event *event)  in perf_event_disable_local()  argument
2451 	event_function_local(event, __perf_event_disable, NULL);  in perf_event_disable_local()
2458 void perf_event_disable(struct perf_event *event)  in perf_event_disable()  argument
2462 	ctx = perf_event_ctx_lock(event);  in perf_event_disable()
2463 	_perf_event_disable(event);  in perf_event_disable()
2464 	perf_event_ctx_unlock(event, ctx);  in perf_event_disable()
2468 void perf_event_disable_inatomic(struct perf_event *event)  in perf_event_disable_inatomic()  argument
2470 	event->pending_disable = 1;  in perf_event_disable_inatomic()
2471 	irq_work_queue(&event->pending_irq);  in perf_event_disable_inatomic()
2476 static void perf_log_throttle(struct perf_event *event, int enable);
2477 static void perf_log_itrace_start(struct perf_event *event);
2480 event_sched_in(struct perf_event *event, struct perf_event_context *ctx)  in event_sched_in()  argument
2482 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in event_sched_in()
2486 	WARN_ON_ONCE(event->ctx != ctx);  in event_sched_in()
2490 	if (event->state <= PERF_EVENT_STATE_OFF)  in event_sched_in()
2493 	WRITE_ONCE(event->oncpu, smp_processor_id());  in event_sched_in()
2500 	perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE);  in event_sched_in()
2507 	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {  in event_sched_in()
2508 		perf_log_throttle(event, 1);  in event_sched_in()
2509 		event->hw.interrupts = 0;  in event_sched_in()
2512 	perf_pmu_disable(event->pmu);  in event_sched_in()
2514 	perf_log_itrace_start(event);  in event_sched_in()
2516 	if (event->pmu->add(event, PERF_EF_START)) {  in event_sched_in()
2517 		perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in event_sched_in()
2518 		event->oncpu = -1;  in event_sched_in()
2523 	if (!is_software_event(event))  in event_sched_in()
2525 	if (event->attr.freq && event->attr.sample_freq)  in event_sched_in()
2528 	if (event->attr.exclusive)  in event_sched_in()
2532 	perf_pmu_enable(event->pmu);  in event_sched_in()
2540 	struct perf_event *event, *partial_group = NULL;  in group_sched_in()  local
2554 	for_each_sibling_event(event, group_event) {  in group_sched_in()
2555 		if (event_sched_in(event, ctx)) {  in group_sched_in()
2556 			partial_group = event;  in group_sched_in()
2570 	for_each_sibling_event(event, group_event) {  in group_sched_in()
2571 		if (event == partial_group)  in group_sched_in()
2574 		event_sched_out(event, ctx);  in group_sched_in()
2586 static int group_can_go_on(struct perf_event *event, int can_add_hw)  in group_can_go_on()  argument
2588 	struct perf_event_pmu_context *epc = event->pmu_ctx;  in group_can_go_on()
2594 	if (event->group_caps & PERF_EV_CAP_SOFTWARE)  in group_can_go_on()
2606 	if (event->attr.exclusive && !list_empty(get_event_list(event)))  in group_can_go_on()
2615 static void add_event_to_ctx(struct perf_event *event,  in add_event_to_ctx()  argument
2618 	list_add_event(event, ctx);  in add_event_to_ctx()
2619 	perf_group_attach(event);  in add_event_to_ctx()
2725 	struct perf_event *event = info;  in __perf_install_in_context()  local
2726 	struct perf_event_context *ctx = event->ctx;  in __perf_install_in_context()
2757 	if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) {  in __perf_install_in_context()
2764 					event->cgrp->css.cgroup);  in __perf_install_in_context()
2770 		add_event_to_ctx(event, ctx);  in __perf_install_in_context()
2771 		ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_install_in_context()
2773 		add_event_to_ctx(event, ctx);  in __perf_install_in_context()
2782 static bool exclusive_event_installable(struct perf_event *event,
2792 			struct perf_event *event,  in perf_install_in_context()  argument
2799 	WARN_ON_ONCE(!exclusive_event_installable(event, ctx));  in perf_install_in_context()
2801 	if (event->cpu != -1)  in perf_install_in_context()
2802 		WARN_ON_ONCE(event->cpu != cpu);  in perf_install_in_context()
2808 	smp_store_release(&event->ctx, ctx);  in perf_install_in_context()
2818 	if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF &&  in perf_install_in_context()
2819 	    ctx->nr_events && !is_cgroup_event(event)) {  in perf_install_in_context()
2825 		add_event_to_ctx(event, ctx);  in perf_install_in_context()
2831 		cpu_function_call(cpu, __perf_install_in_context, event);  in perf_install_in_context()
2873 	if (!task_function_call(task, __perf_install_in_context, event))  in perf_install_in_context()
2895 	add_event_to_ctx(event, ctx);  in perf_install_in_context()
2902 static void __perf_event_enable(struct perf_event *event,  in __perf_event_enable()  argument
2907 	struct perf_event *leader = event->group_leader;  in __perf_event_enable()
2910 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in __perf_event_enable()
2911 	    event->state <= PERF_EVENT_STATE_ERROR)  in __perf_event_enable()
2917 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in __perf_event_enable()
2918 	perf_cgroup_event_enable(event, ctx);  in __perf_event_enable()
2923 	if (!event_filter_match(event)) {  in __perf_event_enable()
2932 	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {  in __perf_event_enable()
2941 	ctx_resched(cpuctx, task_ctx, get_event_type(event));  in __perf_event_enable()
2953 static void _perf_event_enable(struct perf_event *event)  in _perf_event_enable()  argument
2955 	struct perf_event_context *ctx = event->ctx;  in _perf_event_enable()
2958 	if (event->state >= PERF_EVENT_STATE_INACTIVE ||  in _perf_event_enable()
2959 	    event->state <  PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
2972 	if (event->state == PERF_EVENT_STATE_ERROR) {  in _perf_event_enable()
2976 		if (event->event_caps & PERF_EV_CAP_SIBLING &&  in _perf_event_enable()
2977 		    event->group_leader == event)  in _perf_event_enable()
2980 		event->state = PERF_EVENT_STATE_OFF;  in _perf_event_enable()
2984 	event_function_call(event, __perf_event_enable, NULL);  in _perf_event_enable()
2990 void perf_event_enable(struct perf_event *event)  in perf_event_enable()  argument
2994 	ctx = perf_event_ctx_lock(event);  in perf_event_enable()
2995 	_perf_event_enable(event);  in perf_event_enable()
2996 	perf_event_ctx_unlock(event, ctx);  in perf_event_enable()
3001 	struct perf_event	*event;  member
3008 	struct perf_event *event = sd->event;  in __perf_event_stop()  local
3011 	if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in __perf_event_stop()
3021 	if (READ_ONCE(event->oncpu) != smp_processor_id())  in __perf_event_stop()
3024 	event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_stop()
3036 		event->pmu->start(event, 0);  in __perf_event_stop()
3041 static int perf_event_stop(struct perf_event *event, int restart)  in perf_event_stop()  argument
3044 		.event		= event,  in perf_event_stop()
3050 		if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE)  in perf_event_stop()
3061 		ret = cpu_function_call(READ_ONCE(event->oncpu),  in perf_event_stop()
3090 void perf_event_addr_filters_sync(struct perf_event *event)  in perf_event_addr_filters_sync()  argument
3092 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_sync()
3094 	if (!has_addr_filter(event))  in perf_event_addr_filters_sync()
3098 	if (event->addr_filters_gen != event->hw.addr_filters_gen) {  in perf_event_addr_filters_sync()
3099 		event->pmu->addr_filters_sync(event);  in perf_event_addr_filters_sync()
3100 		event->hw.addr_filters_gen = event->addr_filters_gen;  in perf_event_addr_filters_sync()
3106 static int _perf_event_refresh(struct perf_event *event, int refresh)  in _perf_event_refresh()  argument
3111 	if (event->attr.inherit || !is_sampling_event(event))  in _perf_event_refresh()
3114 	atomic_add(refresh, &event->event_limit);  in _perf_event_refresh()
3115 	_perf_event_enable(event);  in _perf_event_refresh()
3123 int perf_event_refresh(struct perf_event *event, int refresh)  in perf_event_refresh()  argument
3128 	ctx = perf_event_ctx_lock(event);  in perf_event_refresh()
3129 	ret = _perf_event_refresh(event, refresh);  in perf_event_refresh()
3130 	perf_event_ctx_unlock(event, ctx);  in perf_event_refresh()
3160 static int perf_event_modify_attr(struct perf_event *event,  in perf_event_modify_attr()  argument
3167 	if (event->attr.type != attr->type)  in perf_event_modify_attr()
3170 	switch (event->attr.type) {  in perf_event_modify_attr()
3179 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_modify_attr()
3181 	mutex_lock(&event->child_mutex);  in perf_event_modify_attr()
3187 	perf_event_modify_copy_attr(&event->attr, attr);  in perf_event_modify_attr()
3188 	err = func(event, attr);  in perf_event_modify_attr()
3191 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_modify_attr()
3198 	mutex_unlock(&event->child_mutex);  in perf_event_modify_attr()
3206 	struct perf_event *event, *tmp;  in __pmu_ctx_sched_out()  local
3222 		list_for_each_entry_safe(event, tmp,  in __pmu_ctx_sched_out()
3225 			group_sched_out(event, ctx);  in __pmu_ctx_sched_out()
3229 		list_for_each_entry_safe(event, tmp,  in __pmu_ctx_sched_out()
3232 			group_sched_out(event, ctx);  in __pmu_ctx_sched_out()
3337 static void __perf_event_sync_stat(struct perf_event *event,  in __perf_event_sync_stat()  argument
3342 	if (!event->attr.inherit_stat)  in __perf_event_sync_stat()
3352 	if (event->state == PERF_EVENT_STATE_ACTIVE)  in __perf_event_sync_stat()
3353 		event->pmu->read(event);  in __perf_event_sync_stat()
3355 	perf_event_update_time(event);  in __perf_event_sync_stat()
3362 	value = local64_xchg(&event->count, value);  in __perf_event_sync_stat()
3365 	swap(event->total_time_enabled, next_event->total_time_enabled);  in __perf_event_sync_stat()
3366 	swap(event->total_time_running, next_event->total_time_running);  in __perf_event_sync_stat()
3371 	perf_event_update_userpage(event);  in __perf_event_sync_stat()
3378 	struct perf_event *event, *next_event;  in perf_event_sync_stat()  local
3385 	event = list_first_entry(&ctx->event_list,  in perf_event_sync_stat()
3391 	while (&event->event_entry != &ctx->event_list &&  in perf_event_sync_stat()
3394 		__perf_event_sync_stat(event, next_event);  in perf_event_sync_stat()
3396 		event = list_next_entry(event, event_entry);  in perf_event_sync_stat()
3665 static void __heap_add(struct min_heap *heap, struct perf_event *event)  in __heap_add()  argument
3669 	if (event) {  in __heap_add()
3670 		itrs[heap->nr] = event;  in __heap_add()
3767 static inline bool event_update_userpage(struct perf_event *event)  in event_update_userpage()  argument
3769 	if (likely(!atomic_read(&event->mmap_count)))  in event_update_userpage()
3772 	perf_event_update_time(event);  in event_update_userpage()
3773 	perf_event_update_userpage(event);  in event_update_userpage()
3780 	struct perf_event *event;  in group_update_userpage()  local
3785 	for_each_sibling_event(event, group_event)  in group_update_userpage()
3786 		event_update_userpage(event);  in group_update_userpage()
3789 static int merge_sched_in(struct perf_event *event, void *data)  in merge_sched_in()  argument
3791 	struct perf_event_context *ctx = event->ctx;  in merge_sched_in()
3794 	if (event->state <= PERF_EVENT_STATE_OFF)  in merge_sched_in()
3797 	if (!event_filter_match(event))  in merge_sched_in()
3800 	if (group_can_go_on(event, *can_add_hw)) {  in merge_sched_in()
3801 		if (!group_sched_in(event, ctx))  in merge_sched_in()
3802 			list_add_tail(&event->active_list, get_event_list(event));  in merge_sched_in()
3805 	if (event->state == PERF_EVENT_STATE_INACTIVE) {  in merge_sched_in()
3807 		if (event->attr.pinned) {  in merge_sched_in()
3808 			perf_cgroup_event_disable(event, ctx);  in merge_sched_in()
3809 			perf_event_set_state(event, PERF_EVENT_STATE_ERROR);  in merge_sched_in()
3813 			event->pmu_ctx->rotate_necessary = 1;  in merge_sched_in()
3814 			cpc = this_cpu_ptr(event->pmu_ctx->pmu->cpu_pmu_context);  in merge_sched_in()
3816 			group_update_userpage(event);  in merge_sched_in()
3991 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)  in perf_calculate_period()  argument
3993 	u64 frequency = event->attr.sample_freq;  in perf_calculate_period()
4067 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)  in perf_adjust_period()  argument
4069 	struct hw_perf_event *hwc = &event->hw;  in perf_adjust_period()
4073 	period = perf_calculate_period(event, nsec, count);  in perf_adjust_period()
4087 			event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_period()
4092 			event->pmu->start(event, PERF_EF_RELOAD);  in perf_adjust_period()
4104 	struct perf_event *event;  in perf_adjust_freq_unthr_context()  local
4119 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_adjust_freq_unthr_context()
4120 		if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_adjust_freq_unthr_context()
4124 		if (!event_filter_match(event))  in perf_adjust_freq_unthr_context()
4127 		perf_pmu_disable(event->pmu);  in perf_adjust_freq_unthr_context()
4129 		hwc = &event->hw;  in perf_adjust_freq_unthr_context()
4133 			perf_log_throttle(event, 1);  in perf_adjust_freq_unthr_context()
4134 			event->pmu->start(event, 0);  in perf_adjust_freq_unthr_context()
4137 		if (!event->attr.freq || !event->attr.sample_freq)  in perf_adjust_freq_unthr_context()
4143 		event->pmu->stop(event, PERF_EF_UPDATE);  in perf_adjust_freq_unthr_context()
4145 		now = local64_read(&event->count);  in perf_adjust_freq_unthr_context()
4157 			perf_adjust_period(event, period, delta, false);  in perf_adjust_freq_unthr_context()
4159 		event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);  in perf_adjust_freq_unthr_context()
4161 		perf_pmu_enable(event->pmu);  in perf_adjust_freq_unthr_context()
4170 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event)  in rotate_ctx()  argument
4179 	perf_event_groups_delete(&ctx->flexible_groups, event);  in rotate_ctx()
4180 	perf_event_groups_insert(&ctx->flexible_groups, event);  in rotate_ctx()
4187 	struct perf_event *event;  in ctx_event_to_rotate()  local
4195 	event = list_first_entry_or_null(&pmu_ctx->flexible_active,  in ctx_event_to_rotate()
4197 	if (event)  in ctx_event_to_rotate()
4208 			event = __node_2_pe(node);  in ctx_event_to_rotate()
4215 		event = __node_2_pe(node);  in ctx_event_to_rotate()
4222 		event = __node_2_pe(node);  in ctx_event_to_rotate()
4231 	return event;  in ctx_event_to_rotate()
4314 static int event_enable_on_exec(struct perf_event *event,  in event_enable_on_exec()  argument
4317 	if (!event->attr.enable_on_exec)  in event_enable_on_exec()
4320 	event->attr.enable_on_exec = 0;  in event_enable_on_exec()
4321 	if (event->state >= PERF_EVENT_STATE_INACTIVE)  in event_enable_on_exec()
4324 	perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE);  in event_enable_on_exec()
4338 	struct perf_event *event;  in perf_event_enable_on_exec()  local
4353 	list_for_each_entry(event, &ctx->event_list, event_entry) {  in perf_event_enable_on_exec()
4354 		enabled |= event_enable_on_exec(event, ctx);  in perf_event_enable_on_exec()
4355 		event_type |= get_event_type(event);  in perf_event_enable_on_exec()
4376 static void perf_remove_from_owner(struct perf_event *event);
4377 static void perf_event_exit_event(struct perf_event *event,
4387 	struct perf_event *event, *next;  in perf_event_remove_on_exec()  local
4396 	list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) {  in perf_event_remove_on_exec()
4397 		if (!event->attr.remove_on_exec)  in perf_event_remove_on_exec()
4400 		if (!is_kernel_event(event))  in perf_event_remove_on_exec()
4401 			perf_remove_from_owner(event);  in perf_event_remove_on_exec()
4405 		perf_event_exit_event(event, ctx);  in perf_event_remove_on_exec()
4421 	struct perf_event *event;  member
4426 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu)  in __perf_event_read_cpu()  argument
4430 	if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) {  in __perf_event_read_cpu()
4449 	struct perf_event *sub, *event = data->event;  in __perf_event_read()  local
4450 	struct perf_event_context *ctx = event->ctx;  in __perf_event_read()
4452 	struct pmu *pmu = event->pmu;  in __perf_event_read()
4467 		update_cgrp_time_from_event(event);  in __perf_event_read()
4470 	perf_event_update_time(event);  in __perf_event_read()
4472 		perf_event_update_sibling_time(event);  in __perf_event_read()
4474 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in __perf_event_read()
4478 		pmu->read(event);  in __perf_event_read()
4485 	pmu->read(event);  in __perf_event_read()
4487 	for_each_sibling_event(sub, event) {  in __perf_event_read()
4503 static inline u64 perf_event_count(struct perf_event *event)  in perf_event_count()  argument
4505 	return local64_read(&event->count) + atomic64_read(&event->child_count);  in perf_event_count()
4508 static void calc_timer_values(struct perf_event *event,  in calc_timer_values()  argument
4516 	ctx_time = perf_event_time_now(event, *now);  in calc_timer_values()
4517 	__perf_update_times(event, ctx_time, enabled, running);  in calc_timer_values()
4528 int perf_event_read_local(struct perf_event *event, u64 *value,  in perf_event_read_local()  argument
4544 	if (event->attr.inherit) {  in perf_event_read_local()
4550 	if ((event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4551 	    event->hw.target != current) {  in perf_event_read_local()
4557 	if (!(event->attach_state & PERF_ATTACH_TASK) &&  in perf_event_read_local()
4558 	    event->cpu != smp_processor_id()) {  in perf_event_read_local()
4564 	if (event->attr.pinned && event->oncpu != smp_processor_id()) {  in perf_event_read_local()
4574 	if (event->oncpu == smp_processor_id())  in perf_event_read_local()
4575 		event->pmu->read(event);  in perf_event_read_local()
4577 	*value = local64_read(&event->count);  in perf_event_read_local()
4581 		calc_timer_values(event, &__now, &__enabled, &__running);  in perf_event_read_local()
4593 static int perf_event_read(struct perf_event *event, bool group)  in perf_event_read()  argument
4595 	enum perf_event_state state = READ_ONCE(event->state);  in perf_event_read()
4614 		event_cpu = READ_ONCE(event->oncpu);  in perf_event_read()
4619 			.event = event,  in perf_event_read()
4625 		event_cpu = __perf_event_read_cpu(event, event_cpu);  in perf_event_read()
4642 		struct perf_event_context *ctx = event->ctx;  in perf_event_read()
4646 		state = event->state;  in perf_event_read()
4658 			update_cgrp_time_from_event(event);  in perf_event_read()
4661 		perf_event_update_time(event);  in perf_event_read()
4663 			perf_event_update_sibling_time(event);  in perf_event_read()
4734 find_get_context(struct task_struct *task, struct perf_event *event)  in find_get_context()  argument
4743 		err = perf_allow_cpu(&event->attr);  in find_get_context()
4747 		cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in find_get_context()
4808 		     struct perf_event *event)  in find_get_pmu_context()  argument
4816 		cpc = per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);  in find_get_pmu_context()
4836 	if (event->attach_state & PERF_ATTACH_TASK_DATA) {  in find_get_pmu_context()
4929 static void perf_event_free_filter(struct perf_event *event);
4933 	struct perf_event *event = container_of(head, typeof(*event), rcu_head);  in free_event_rcu()  local
4935 	if (event->ns)  in free_event_rcu()
4936 		put_pid_ns(event->ns);  in free_event_rcu()
4937 	perf_event_free_filter(event);  in free_event_rcu()
4938 	kmem_cache_free(perf_event_cache, event);  in free_event_rcu()
4941 static void ring_buffer_attach(struct perf_event *event,
4944 static void detach_sb_event(struct perf_event *event)  in detach_sb_event()  argument
4946 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in detach_sb_event()
4949 	list_del_rcu(&event->sb_list);  in detach_sb_event()
4953 static bool is_sb_event(struct perf_event *event)  in is_sb_event()  argument
4955 	struct perf_event_attr *attr = &event->attr;  in is_sb_event()
4957 	if (event->parent)  in is_sb_event()
4960 	if (event->attach_state & PERF_ATTACH_TASK)  in is_sb_event()
4972 static void unaccount_pmu_sb_event(struct perf_event *event)  in unaccount_pmu_sb_event()  argument
4974 	if (is_sb_event(event))  in unaccount_pmu_sb_event()
4975 		detach_sb_event(event);  in unaccount_pmu_sb_event()
5000 static void unaccount_event(struct perf_event *event)  in unaccount_event()  argument
5004 	if (event->parent)  in unaccount_event()
5007 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in unaccount_event()
5009 	if (event->attr.mmap || event->attr.mmap_data)  in unaccount_event()
5011 	if (event->attr.build_id)  in unaccount_event()
5013 	if (event->attr.comm)  in unaccount_event()
5015 	if (event->attr.namespaces)  in unaccount_event()
5017 	if (event->attr.cgroup)  in unaccount_event()
5019 	if (event->attr.task)  in unaccount_event()
5021 	if (event->attr.freq)  in unaccount_event()
5023 	if (event->attr.context_switch) {  in unaccount_event()
5027 	if (is_cgroup_event(event))  in unaccount_event()
5029 	if (has_branch_stack(event))  in unaccount_event()
5031 	if (event->attr.ksymbol)  in unaccount_event()
5033 	if (event->attr.bpf_event)  in unaccount_event()
5035 	if (event->attr.text_poke)  in unaccount_event()
5043 	unaccount_pmu_sb_event(event);  in unaccount_event()
5066 static int exclusive_event_init(struct perf_event *event)  in exclusive_event_init()  argument
5068 	struct pmu *pmu = event->pmu;  in exclusive_event_init()
5086 	if (event->attach_state & PERF_ATTACH_TASK) {  in exclusive_event_init()
5097 static void exclusive_event_destroy(struct perf_event *event)  in exclusive_event_destroy()  argument
5099 	struct pmu *pmu = event->pmu;  in exclusive_event_destroy()
5105 	if (event->attach_state & PERF_ATTACH_TASK)  in exclusive_event_destroy()
5121 static bool exclusive_event_installable(struct perf_event *event,  in exclusive_event_installable()  argument
5125 	struct pmu *pmu = event->pmu;  in exclusive_event_installable()
5133 		if (exclusive_event_match(iter_event, event))  in exclusive_event_installable()
5140 static void perf_addr_filters_splice(struct perf_event *event,
5143 static void _free_event(struct perf_event *event)  in _free_event()  argument
5145 	irq_work_sync(&event->pending_irq);  in _free_event()
5147 	unaccount_event(event);  in _free_event()
5149 	security_perf_event_free(event);  in _free_event()
5151 	if (event->rb) {  in _free_event()
5158 		mutex_lock(&event->mmap_mutex);  in _free_event()
5159 		ring_buffer_attach(event, NULL);  in _free_event()
5160 		mutex_unlock(&event->mmap_mutex);  in _free_event()
5163 	if (is_cgroup_event(event))  in _free_event()
5164 		perf_detach_cgroup(event);  in _free_event()
5166 	if (!event->parent) {  in _free_event()
5167 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in _free_event()
5171 	perf_event_free_bpf_prog(event);  in _free_event()
5172 	perf_addr_filters_splice(event, NULL);  in _free_event()
5173 	kfree(event->addr_filter_ranges);  in _free_event()
5175 	if (event->destroy)  in _free_event()
5176 		event->destroy(event);  in _free_event()
5182 	if (event->hw.target)  in _free_event()
5183 		put_task_struct(event->hw.target);  in _free_event()
5185 	if (event->pmu_ctx)  in _free_event()
5186 		put_pmu_ctx(event->pmu_ctx);  in _free_event()
5192 	if (event->ctx)  in _free_event()
5193 		put_ctx(event->ctx);  in _free_event()
5195 	exclusive_event_destroy(event);  in _free_event()
5196 	module_put(event->pmu->module);  in _free_event()
5198 	call_rcu(&event->rcu_head, free_event_rcu);  in _free_event()
5205 static void free_event(struct perf_event *event)  in free_event()  argument
5207 	if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,  in free_event()
5209 				atomic_long_read(&event->refcount), event)) {  in free_event()
5214 	_free_event(event);  in free_event()
5220 static void perf_remove_from_owner(struct perf_event *event)  in perf_remove_from_owner()  argument
5231 	owner = READ_ONCE(event->owner);  in perf_remove_from_owner()
5259 		if (event->owner) {  in perf_remove_from_owner()
5260 			list_del_init(&event->owner_entry);  in perf_remove_from_owner()
5261 			smp_store_release(&event->owner, NULL);  in perf_remove_from_owner()
5268 static void put_event(struct perf_event *event)  in put_event()  argument
5270 	if (!atomic_long_dec_and_test(&event->refcount))  in put_event()
5273 	_free_event(event);  in put_event()
5281 int perf_event_release_kernel(struct perf_event *event)  in perf_event_release_kernel()  argument
5283 	struct perf_event_context *ctx = event->ctx;  in perf_event_release_kernel()
5292 		WARN_ON_ONCE(event->attach_state &  in perf_event_release_kernel()
5297 	if (!is_kernel_event(event))  in perf_event_release_kernel()
5298 		perf_remove_from_owner(event);  in perf_event_release_kernel()
5300 	ctx = perf_event_ctx_lock(event);  in perf_event_release_kernel()
5314 	perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD);  in perf_event_release_kernel()
5316 	perf_event_ctx_unlock(event, ctx);  in perf_event_release_kernel()
5319 	mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5320 	list_for_each_entry(child, &event->child_list, child_list) {  in perf_event_release_kernel()
5342 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5344 		mutex_lock(&event->child_mutex);  in perf_event_release_kernel()
5351 		tmp = list_first_entry_or_null(&event->child_list,  in perf_event_release_kernel()
5360 			put_event(event);  in perf_event_release_kernel()
5363 		mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5368 	mutex_unlock(&event->child_mutex);  in perf_event_release_kernel()
5385 	put_event(event); /* Must be the 'last' reference */  in perf_event_release_kernel()
5399 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)  in __perf_event_read_value()  argument
5407 	mutex_lock(&event->child_mutex);  in __perf_event_read_value()
5409 	(void)perf_event_read(event, false);  in __perf_event_read_value()
5410 	total += perf_event_count(event);  in __perf_event_read_value()
5412 	*enabled += event->total_time_enabled +  in __perf_event_read_value()
5413 			atomic64_read(&event->child_total_time_enabled);  in __perf_event_read_value()
5414 	*running += event->total_time_running +  in __perf_event_read_value()
5415 			atomic64_read(&event->child_total_time_running);  in __perf_event_read_value()
5417 	list_for_each_entry(child, &event->child_list, child_list) {  in __perf_event_read_value()
5423 	mutex_unlock(&event->child_mutex);  in __perf_event_read_value()
5428 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)  in perf_event_read_value()  argument
5433 	ctx = perf_event_ctx_lock(event);  in perf_event_read_value()
5434 	count = __perf_event_read_value(event, enabled, running);  in perf_event_read_value()
5435 	perf_event_ctx_unlock(event, ctx);  in perf_event_read_value()
5520 static int perf_read_group(struct perf_event *event,  in perf_read_group()  argument
5523 	struct perf_event *leader = event->group_leader, *child;  in perf_read_group()
5530 	values = kzalloc(event->read_size, GFP_KERNEL);  in perf_read_group()
5550 	ret = event->read_size;  in perf_read_group()
5551 	if (copy_to_user(buf, values, event->read_size))  in perf_read_group()
5562 static int perf_read_one(struct perf_event *event,  in perf_read_one()  argument
5569 	values[n++] = __perf_event_read_value(event, &enabled, &running);  in perf_read_one()
5575 		values[n++] = primary_event_id(event);  in perf_read_one()
5577 		values[n++] = atomic64_read(&event->lost_samples);  in perf_read_one()
5585 static bool is_event_hup(struct perf_event *event)  in is_event_hup()  argument
5589 	if (event->state > PERF_EVENT_STATE_EXIT)  in is_event_hup()
5592 	mutex_lock(&event->child_mutex);  in is_event_hup()
5593 	no_children = list_empty(&event->child_list);  in is_event_hup()
5594 	mutex_unlock(&event->child_mutex);  in is_event_hup()
5602 __perf_read(struct perf_event *event, char __user *buf, size_t count)  in __perf_read()  argument
5604 	u64 read_format = event->attr.read_format;  in __perf_read()
5612 	if (event->state == PERF_EVENT_STATE_ERROR)  in __perf_read()
5615 	if (count < event->read_size)  in __perf_read()
5618 	WARN_ON_ONCE(event->ctx->parent_ctx);  in __perf_read()
5620 		ret = perf_read_group(event, read_format, buf);  in __perf_read()
5622 		ret = perf_read_one(event, read_format, buf);  in __perf_read()
5630 	struct perf_event *event = file->private_data;  in perf_read()  local
5634 	ret = security_perf_event_read(event);  in perf_read()
5638 	ctx = perf_event_ctx_lock(event);  in perf_read()
5639 	ret = __perf_read(event, buf, count);  in perf_read()
5640 	perf_event_ctx_unlock(event, ctx);  in perf_read()
5647 	struct perf_event *event = file->private_data;  in perf_poll()  local
5651 	poll_wait(file, &event->waitq, wait);  in perf_poll()
5653 	if (is_event_hup(event))  in perf_poll()
5660 	mutex_lock(&event->mmap_mutex);  in perf_poll()
5661 	rb = event->rb;  in perf_poll()
5664 	mutex_unlock(&event->mmap_mutex);  in perf_poll()
5668 static void _perf_event_reset(struct perf_event *event)  in _perf_event_reset()  argument
5670 	(void)perf_event_read(event, false);  in _perf_event_reset()
5671 	local64_set(&event->count, 0);  in _perf_event_reset()
5672 	perf_event_update_userpage(event);  in _perf_event_reset()
5676 u64 perf_event_pause(struct perf_event *event, bool reset)  in perf_event_pause()  argument
5681 	ctx = perf_event_ctx_lock(event);  in perf_event_pause()
5682 	WARN_ON_ONCE(event->attr.inherit);  in perf_event_pause()
5683 	_perf_event_disable(event);  in perf_event_pause()
5684 	count = local64_read(&event->count);  in perf_event_pause()
5686 		local64_set(&event->count, 0);  in perf_event_pause()
5687 	perf_event_ctx_unlock(event, ctx);  in perf_event_pause()
5699 static void perf_event_for_each_child(struct perf_event *event,  in perf_event_for_each_child()  argument
5704 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_event_for_each_child()
5706 	mutex_lock(&event->child_mutex);  in perf_event_for_each_child()
5707 	func(event);  in perf_event_for_each_child()
5708 	list_for_each_entry(child, &event->child_list, child_list)  in perf_event_for_each_child()
5710 	mutex_unlock(&event->child_mutex);  in perf_event_for_each_child()
5713 static void perf_event_for_each(struct perf_event *event,  in perf_event_for_each()  argument
5716 	struct perf_event_context *ctx = event->ctx;  in perf_event_for_each()
5721 	event = event->group_leader;  in perf_event_for_each()
5723 	perf_event_for_each_child(event, func);  in perf_event_for_each()
5724 	for_each_sibling_event(sibling, event)  in perf_event_for_each()
5728 static void __perf_event_period(struct perf_event *event,  in __perf_event_period()  argument
5736 	if (event->attr.freq) {  in __perf_event_period()
5737 		event->attr.sample_freq = value;  in __perf_event_period()
5739 		event->attr.sample_period = value;  in __perf_event_period()
5740 		event->hw.sample_period = value;  in __perf_event_period()
5743 	active = (event->state == PERF_EVENT_STATE_ACTIVE);  in __perf_event_period()
5745 		perf_pmu_disable(event->pmu);  in __perf_event_period()
5750 		if (event->hw.interrupts == MAX_INTERRUPTS) {  in __perf_event_period()
5751 			event->hw.interrupts = 0;  in __perf_event_period()
5752 			perf_log_throttle(event, 1);  in __perf_event_period()
5754 		event->pmu->stop(event, PERF_EF_UPDATE);  in __perf_event_period()
5757 	local64_set(&event->hw.period_left, 0);  in __perf_event_period()
5760 		event->pmu->start(event, PERF_EF_RELOAD);  in __perf_event_period()
5761 		perf_pmu_enable(event->pmu);  in __perf_event_period()
5765 static int perf_event_check_period(struct perf_event *event, u64 value)  in perf_event_check_period()  argument
5767 	return event->pmu->check_period(event, value);  in perf_event_check_period()
5770 static int _perf_event_period(struct perf_event *event, u64 value)  in _perf_event_period()  argument
5772 	if (!is_sampling_event(event))  in _perf_event_period()
5778 	if (event->attr.freq && value > sysctl_perf_event_sample_rate)  in _perf_event_period()
5781 	if (perf_event_check_period(event, value))  in _perf_event_period()
5784 	if (!event->attr.freq && (value & (1ULL << 63)))  in _perf_event_period()
5787 	event_function_call(event, __perf_event_period, &value);  in _perf_event_period()
5792 int perf_event_period(struct perf_event *event, u64 value)  in perf_event_period()  argument
5797 	ctx = perf_event_ctx_lock(event);  in perf_event_period()
5798 	ret = _perf_event_period(event, value);  in perf_event_period()
5799 	perf_event_ctx_unlock(event, ctx);  in perf_event_period()
5821 static int perf_event_set_output(struct perf_event *event,
5823 static int perf_event_set_filter(struct perf_event *event, void __user *arg);
5827 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg)  in _perf_ioctl()  argument
5844 		return _perf_event_refresh(event, arg);  in _perf_ioctl()
5853 		return _perf_event_period(event, value);  in _perf_ioctl()
5857 		u64 id = primary_event_id(event);  in _perf_ioctl()
5874 			ret = perf_event_set_output(event, output_event);  in _perf_ioctl()
5877 			ret = perf_event_set_output(event, NULL);  in _perf_ioctl()
5883 		return perf_event_set_filter(event, (void __user *)arg);  in _perf_ioctl()
5894 		err = perf_event_set_bpf_prog(event, prog, 0);  in _perf_ioctl()
5907 		rb = rcu_dereference(event->rb);  in _perf_ioctl()
5918 		return perf_event_query_prog_array(event, (void __user *)arg);  in _perf_ioctl()
5928 		return perf_event_modify_attr(event,  &new_attr);  in _perf_ioctl()
5935 		perf_event_for_each(event, func);  in _perf_ioctl()
5937 		perf_event_for_each_child(event, func);  in _perf_ioctl()
5944 	struct perf_event *event = file->private_data;  in perf_ioctl()  local
5949 	ret = security_perf_event_write(event);  in perf_ioctl()
5953 	ctx = perf_event_ctx_lock(event);  in perf_ioctl()
5954 	ret = _perf_ioctl(event, cmd, arg);  in perf_ioctl()
5955 	perf_event_ctx_unlock(event, ctx);  in perf_ioctl()
5985 	struct perf_event *event;  in perf_event_task_enable()  local
5988 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_enable()
5989 		ctx = perf_event_ctx_lock(event);  in perf_event_task_enable()
5990 		perf_event_for_each_child(event, _perf_event_enable);  in perf_event_task_enable()
5991 		perf_event_ctx_unlock(event, ctx);  in perf_event_task_enable()
6001 	struct perf_event *event;  in perf_event_task_disable()  local
6004 	list_for_each_entry(event, &current->perf_event_list, owner_entry) {  in perf_event_task_disable()
6005 		ctx = perf_event_ctx_lock(event);  in perf_event_task_disable()
6006 		perf_event_for_each_child(event, _perf_event_disable);  in perf_event_task_disable()
6007 		perf_event_ctx_unlock(event, ctx);  in perf_event_task_disable()
6014 static int perf_event_index(struct perf_event *event)  in perf_event_index()  argument
6016 	if (event->hw.state & PERF_HES_STOPPED)  in perf_event_index()
6019 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_event_index()
6022 	return event->pmu->event_idx(event);  in perf_event_index()
6025 static void perf_event_init_userpage(struct perf_event *event)  in perf_event_init_userpage()  argument
6031 	rb = rcu_dereference(event->rb);  in perf_event_init_userpage()
6048 	struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now)  in arch_perf_update_userpage()  argument
6057 void perf_event_update_userpage(struct perf_event *event)  in perf_event_update_userpage()  argument
6064 	rb = rcu_dereference(event->rb);  in perf_event_update_userpage()
6077 	calc_timer_values(event, &now, &enabled, &running);  in perf_event_update_userpage()
6087 	userpg->index = perf_event_index(event);  in perf_event_update_userpage()
6088 	userpg->offset = perf_event_count(event);  in perf_event_update_userpage()
6090 		userpg->offset -= local64_read(&event->hw.prev_count);  in perf_event_update_userpage()
6093 			atomic64_read(&event->child_total_time_enabled);  in perf_event_update_userpage()
6096 			atomic64_read(&event->child_total_time_running);  in perf_event_update_userpage()
6098 	arch_perf_update_userpage(event, userpg, now);  in perf_event_update_userpage()
6110 	struct perf_event *event = vmf->vma->vm_file->private_data;  in perf_mmap_fault()  local
6121 	rb = rcu_dereference(event->rb);  in perf_mmap_fault()
6143 static void ring_buffer_attach(struct perf_event *event,  in ring_buffer_attach()  argument
6149 	WARN_ON_ONCE(event->parent);  in ring_buffer_attach()
6151 	if (event->rb) {  in ring_buffer_attach()
6156 		WARN_ON_ONCE(event->rcu_pending);  in ring_buffer_attach()
6158 		old_rb = event->rb;  in ring_buffer_attach()
6160 		list_del_rcu(&event->rb_entry);  in ring_buffer_attach()
6163 		event->rcu_batches = get_state_synchronize_rcu();  in ring_buffer_attach()
6164 		event->rcu_pending = 1;  in ring_buffer_attach()
6168 		if (event->rcu_pending) {  in ring_buffer_attach()
6169 			cond_synchronize_rcu(event->rcu_batches);  in ring_buffer_attach()
6170 			event->rcu_pending = 0;  in ring_buffer_attach()
6174 		list_add_rcu(&event->rb_entry, &rb->event_list);  in ring_buffer_attach()
6188 	if (has_aux(event))  in ring_buffer_attach()
6189 		perf_event_stop(event, 0);  in ring_buffer_attach()
6191 	rcu_assign_pointer(event->rb, rb);  in ring_buffer_attach()
6200 		wake_up_all(&event->waitq);  in ring_buffer_attach()
6204 static void ring_buffer_wakeup(struct perf_event *event)  in ring_buffer_wakeup()  argument
6208 	if (event->parent)  in ring_buffer_wakeup()
6209 		event = event->parent;  in ring_buffer_wakeup()
6212 	rb = rcu_dereference(event->rb);  in ring_buffer_wakeup()
6214 		list_for_each_entry_rcu(event, &rb->event_list, rb_entry)  in ring_buffer_wakeup()
6215 			wake_up_all(&event->waitq);  in ring_buffer_wakeup()
6220 struct perf_buffer *ring_buffer_get(struct perf_event *event)  in ring_buffer_get()  argument
6224 	if (event->parent)  in ring_buffer_get()
6225 		event = event->parent;  in ring_buffer_get()
6228 	rb = rcu_dereference(event->rb);  in ring_buffer_get()
6250 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_open()  local
6252 	atomic_inc(&event->mmap_count);  in perf_mmap_open()
6253 	atomic_inc(&event->rb->mmap_count);  in perf_mmap_open()
6256 		atomic_inc(&event->rb->aux_mmap_count);  in perf_mmap_open()
6258 	if (event->pmu->event_mapped)  in perf_mmap_open()
6259 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap_open()
6262 static void perf_pmu_output_stop(struct perf_event *event);
6274 	struct perf_event *event = vma->vm_file->private_data;  in perf_mmap_close()  local
6275 	struct perf_buffer *rb = ring_buffer_get(event);  in perf_mmap_close()
6281 	if (event->pmu->event_unmapped)  in perf_mmap_close()
6282 		event->pmu->event_unmapped(event, vma->vm_mm);  in perf_mmap_close()
6290 	    atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) {  in perf_mmap_close()
6297 		perf_pmu_output_stop(event);  in perf_mmap_close()
6307 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6313 	if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))  in perf_mmap_close()
6316 	ring_buffer_attach(event, NULL);  in perf_mmap_close()
6317 	mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6330 	list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {  in perf_mmap_close()
6331 		if (!atomic_long_inc_not_zero(&event->refcount)) {  in perf_mmap_close()
6340 		mutex_lock(&event->mmap_mutex);  in perf_mmap_close()
6351 		if (event->rb == rb)  in perf_mmap_close()
6352 			ring_buffer_attach(event, NULL);  in perf_mmap_close()
6354 		mutex_unlock(&event->mmap_mutex);  in perf_mmap_close()
6355 		put_event(event);  in perf_mmap_close()
6392 	struct perf_event *event = file->private_data;  in perf_mmap()  local
6407 	if (event->cpu == -1 && event->attr.inherit)  in perf_mmap()
6413 	ret = security_perf_event_read(event);  in perf_mmap()
6429 		if (!event->rb)  in perf_mmap()
6434 		mutex_lock(&event->mmap_mutex);  in perf_mmap()
6437 		rb = event->rb;  in perf_mmap()
6489 	WARN_ON_ONCE(event->ctx->parent_ctx);  in perf_mmap()
6491 	mutex_lock(&event->mmap_mutex);  in perf_mmap()
6492 	if (event->rb) {  in perf_mmap()
6493 		if (data_page_nr(event->rb) != nr_pages) {  in perf_mmap()
6498 		if (!atomic_inc_not_zero(&event->rb->mmap_count)) {  in perf_mmap()
6503 			ring_buffer_attach(event, NULL);  in perf_mmap()
6504 			mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6550 	WARN_ON(!rb && event->rb);  in perf_mmap()
6557 			      event->attr.watermark ? event->attr.wakeup_watermark : 0,  in perf_mmap()
6558 			      event->cpu, flags);  in perf_mmap()
6569 		ring_buffer_attach(event, rb);  in perf_mmap()
6571 		perf_event_update_time(event);  in perf_mmap()
6572 		perf_event_init_userpage(event);  in perf_mmap()
6573 		perf_event_update_userpage(event);  in perf_mmap()
6575 		ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages,  in perf_mmap()
6576 				   event->attr.aux_watermark, flags);  in perf_mmap()
6586 		atomic_inc(&event->mmap_count);  in perf_mmap()
6591 	mutex_unlock(&event->mmap_mutex);  in perf_mmap()
6600 	if (event->pmu->event_mapped)  in perf_mmap()
6601 		event->pmu->event_mapped(event, vma->vm_mm);  in perf_mmap()
6609 	struct perf_event *event = filp->private_data;  in perf_fasync()  local
6613 	retval = fasync_helper(fd, filp, on, &event->fasync);  in perf_fasync()
6640 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)  in perf_event_fasync()  argument
6643 	if (event->parent)  in perf_event_fasync()
6644 		event = event->parent;  in perf_event_fasync()
6645 	return &event->fasync;  in perf_event_fasync()
6648 void perf_event_wakeup(struct perf_event *event)  in perf_event_wakeup()  argument
6650 	ring_buffer_wakeup(event);  in perf_event_wakeup()
6652 	if (event->pending_kill) {  in perf_event_wakeup()
6653 		kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);  in perf_event_wakeup()
6654 		event->pending_kill = 0;  in perf_event_wakeup()
6658 static void perf_sigtrap(struct perf_event *event)  in perf_sigtrap()  argument
6665 	if (WARN_ON_ONCE(event->ctx->task != current))  in perf_sigtrap()
6675 	send_sig_perf((void __user *)event->pending_addr,  in perf_sigtrap()
6676 		      event->orig_type, event->attr.sig_data);  in perf_sigtrap()
6682 static void __perf_pending_irq(struct perf_event *event)  in __perf_pending_irq()  argument
6684 	int cpu = READ_ONCE(event->oncpu);  in __perf_pending_irq()
6697 		if (event->pending_sigtrap) {  in __perf_pending_irq()
6698 			event->pending_sigtrap = 0;  in __perf_pending_irq()
6699 			perf_sigtrap(event);  in __perf_pending_irq()
6700 			local_dec(&event->ctx->nr_pending);  in __perf_pending_irq()
6702 		if (event->pending_disable) {  in __perf_pending_irq()
6703 			event->pending_disable = 0;  in __perf_pending_irq()
6704 			perf_event_disable_local(event);  in __perf_pending_irq()
6729 	irq_work_queue_on(&event->pending_irq, cpu);  in __perf_pending_irq()
6734 	struct perf_event *event = container_of(entry, struct perf_event, pending_irq);  in perf_pending_irq()  local
6747 	if (event->pending_wakeup) {  in perf_pending_irq()
6748 		event->pending_wakeup = 0;  in perf_pending_irq()
6749 		perf_event_wakeup(event);  in perf_pending_irq()
6752 	__perf_pending_irq(event);  in perf_pending_irq()
6760 	struct perf_event *event = container_of(head, struct perf_event, pending_task);  in perf_pending_task()  local
6770 	if (event->pending_work) {  in perf_pending_task()
6771 		event->pending_work = 0;  in perf_pending_task()
6772 		perf_sigtrap(event);  in perf_pending_task()
6773 		local_dec(&event->ctx->nr_pending);  in perf_pending_task()
6780 	put_event(event);  in perf_pending_task()
6954 static unsigned long perf_prepare_sample_aux(struct perf_event *event,  in perf_prepare_sample_aux()  argument
6958 	struct perf_event *sampler = event->aux_event;  in perf_prepare_sample_aux()
6993                                  struct perf_event *event,  in perf_pmu_snapshot_aux()  argument
7017 	ret = event->pmu->snapshot_aux(event, handle, size);  in perf_pmu_snapshot_aux()
7026 static void perf_aux_sample_output(struct perf_event *event,  in perf_aux_sample_output()  argument
7030 	struct perf_event *sampler = event->aux_event;  in perf_aux_sample_output()
7079 					 struct perf_event *event,  in __perf_event_header__init_id()  argument
7082 	data->type = event->attr.sample_type;  in __perf_event_header__init_id()
7087 		data->tid_entry.pid = perf_event_pid(event, current);  in __perf_event_header__init_id()
7088 		data->tid_entry.tid = perf_event_tid(event, current);  in __perf_event_header__init_id()
7092 		data->time = perf_event_clock(event);  in __perf_event_header__init_id()
7095 		data->id = primary_event_id(event);  in __perf_event_header__init_id()
7098 		data->stream_id = event->id;  in __perf_event_header__init_id()
7108 				struct perf_event *event)  in perf_event_header__init_id()  argument
7110 	if (event->attr.sample_id_all) {  in perf_event_header__init_id()
7111 		header->size += event->id_header_size;  in perf_event_header__init_id()
7112 		__perf_event_header__init_id(data, event, event->attr.sample_type);  in perf_event_header__init_id()
7140 void perf_event__output_id_sample(struct perf_event *event,  in perf_event__output_id_sample()  argument
7144 	if (event->attr.sample_id_all)  in perf_event__output_id_sample()
7149 				 struct perf_event *event,  in perf_output_read_one()  argument
7152 	u64 read_format = event->attr.read_format;  in perf_output_read_one()
7156 	values[n++] = perf_event_count(event);  in perf_output_read_one()
7159 			atomic64_read(&event->child_total_time_enabled);  in perf_output_read_one()
7163 			atomic64_read(&event->child_total_time_running);  in perf_output_read_one()
7166 		values[n++] = primary_event_id(event);  in perf_output_read_one()
7168 		values[n++] = atomic64_read(&event->lost_samples);  in perf_output_read_one()
7174 			    struct perf_event *event,  in perf_output_read_group()  argument
7177 	struct perf_event *leader = event->group_leader, *sub;  in perf_output_read_group()
7178 	u64 read_format = event->attr.read_format;  in perf_output_read_group()
7197 	if ((leader != event) &&  in perf_output_read_group()
7212 		if ((sub != event) &&  in perf_output_read_group()
7239 			     struct perf_event *event)  in perf_output_read()  argument
7242 	u64 read_format = event->attr.read_format;  in perf_output_read()
7254 		calc_timer_values(event, &now, &enabled, &running);  in perf_output_read()
7256 	if (event->attr.read_format & PERF_FORMAT_GROUP)  in perf_output_read()
7257 		perf_output_read_group(handle, event, enabled, running);  in perf_output_read()
7259 		perf_output_read_one(handle, event, enabled, running);  in perf_output_read()
7265 			struct perf_event *event)  in perf_output_sample()  argument
7299 		perf_output_read(handle, event);  in perf_output_sample()
7350 			if (branch_sample_hw_index(event))  in perf_output_sample()
7372 			u64 mask = event->attr.sample_regs_user;  in perf_output_sample()
7403 			u64 mask = event->attr.sample_regs_intr;  in perf_output_sample()
7427 			perf_aux_sample_output(event, handle, data);  in perf_output_sample()
7430 	if (!event->attr.watermark) {  in perf_output_sample()
7431 		int wakeup_events = event->attr.wakeup_events;  in perf_output_sample()
7574 perf_callchain(struct perf_event *event, struct pt_regs *regs)  in perf_callchain()  argument
7576 	bool kernel = !event->attr.exclude_callchain_kernel;  in perf_callchain()
7577 	bool user   = !event->attr.exclude_callchain_user;  in perf_callchain()
7579 	bool crosstask = event->ctx->task && event->ctx->task != current;  in perf_callchain()
7580 	const u32 max_stack = event->attr.sample_max_stack;  in perf_callchain()
7597 			 struct perf_event *event,  in perf_prepare_sample()  argument
7600 	u64 sample_type = event->attr.sample_type;  in perf_prepare_sample()
7618 		data->type = event->attr.sample_type;  in perf_prepare_sample()
7622 	__perf_event_header__init_id(data, event, filtered_sample_type);  in perf_prepare_sample()
7630 		perf_sample_save_callchain(data, event, regs);  in perf_prepare_sample()
7657 			u64 mask = event->attr.sample_regs_user;  in perf_prepare_sample()
7672 		u16 stack_size = event->attr.sample_stack_user;  in perf_prepare_sample()
7673 		u16 header_size = perf_sample_data_size(data, event);  in perf_prepare_sample()
7719 			u64 mask = event->attr.sample_regs_intr;  in perf_prepare_sample()
7761 		u16 header_size = perf_sample_data_size(data, event);  in perf_prepare_sample()
7772 			     event->attr.aux_sample_size);  in perf_prepare_sample()
7774 		size = perf_prepare_sample_aux(event, data, size);  in perf_prepare_sample()
7784 			 struct perf_event *event,  in perf_prepare_header()  argument
7788 	header->size = perf_sample_data_size(data, event);  in perf_prepare_header()
7803 __perf_event_output(struct perf_event *event,  in __perf_event_output()  argument
7818 	perf_prepare_sample(data, event, regs);  in __perf_event_output()
7819 	perf_prepare_header(&header, data, event, regs);  in __perf_event_output()
7821 	err = output_begin(&handle, data, event, header.size);  in __perf_event_output()
7825 	perf_output_sample(&handle, &header, data, event);  in __perf_event_output()
7835 perf_event_output_forward(struct perf_event *event,  in perf_event_output_forward()  argument
7839 	__perf_event_output(event, data, regs, perf_output_begin_forward);  in perf_event_output_forward()
7843 perf_event_output_backward(struct perf_event *event,  in perf_event_output_backward()  argument
7847 	__perf_event_output(event, data, regs, perf_output_begin_backward);  in perf_event_output_backward()
7851 perf_event_output(struct perf_event *event,  in perf_event_output()  argument
7855 	return __perf_event_output(event, data, regs, perf_output_begin);  in perf_event_output()
7870 perf_event_read_event(struct perf_event *event,  in perf_event_read_event()  argument
7879 			.size = sizeof(read_event) + event->read_size,  in perf_event_read_event()
7881 		.pid = perf_event_pid(event, task),  in perf_event_read_event()
7882 		.tid = perf_event_tid(event, task),  in perf_event_read_event()
7886 	perf_event_header__init_id(&read_event.header, &sample, event);  in perf_event_read_event()
7887 	ret = perf_output_begin(&handle, &sample, event, read_event.header.size);  in perf_event_read_event()
7892 	perf_output_read(&handle, event);  in perf_event_read_event()
7893 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_read_event()
7898 typedef void (perf_iterate_f)(struct perf_event *event, void *data);
7905 	struct perf_event *event;  in perf_iterate_ctx()  local
7907 	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {  in perf_iterate_ctx()
7909 			if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_ctx()
7911 			if (!event_filter_match(event))  in perf_iterate_ctx()
7915 		output(event, data);  in perf_iterate_ctx()
7922 	struct perf_event *event;  in perf_iterate_sb_cpu()  local
7924 	list_for_each_entry_rcu(event, &pel->list, sb_list) {  in perf_iterate_sb_cpu()
7930 		if (!smp_load_acquire(&event->ctx))  in perf_iterate_sb_cpu()
7933 		if (event->state < PERF_EVENT_STATE_INACTIVE)  in perf_iterate_sb_cpu()
7935 		if (!event_filter_match(event))  in perf_iterate_sb_cpu()
7937 		output(event, data);  in perf_iterate_sb_cpu()
7980 static void perf_event_addr_filters_exec(struct perf_event *event, void *data)  in perf_event_addr_filters_exec()  argument
7982 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_exec()
7987 	if (!has_addr_filter(event))  in perf_event_addr_filters_exec()
7993 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_exec()
7994 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_exec()
8002 		event->addr_filters_gen++;  in perf_event_addr_filters_exec()
8006 		perf_event_stop(event, 1);  in perf_event_addr_filters_exec()
8030 static void __perf_event_output_stop(struct perf_event *event, void *data)  in __perf_event_output_stop()  argument
8032 	struct perf_event *parent = event->parent;  in __perf_event_output_stop()
8036 		.event	= event,  in __perf_event_output_stop()
8039 	if (!has_aux(event))  in __perf_event_output_stop()
8043 		parent = event;  in __perf_event_output_stop()
8061 	struct perf_event *event = info;  in __perf_pmu_output_stop()  local
8064 		.rb	= event->rb,  in __perf_pmu_output_stop()
8077 static void perf_pmu_output_stop(struct perf_event *event)  in perf_pmu_output_stop()  argument
8084 	list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) {  in perf_pmu_output_stop()
8098 		err = cpu_function_call(cpu, __perf_pmu_output_stop, event);  in perf_pmu_output_stop()
8128 static int perf_event_task_match(struct perf_event *event)  in perf_event_task_match()  argument
8130 	return event->attr.comm  || event->attr.mmap ||  in perf_event_task_match()
8131 	       event->attr.mmap2 || event->attr.mmap_data ||  in perf_event_task_match()
8132 	       event->attr.task;  in perf_event_task_match()
8135 static void perf_event_task_output(struct perf_event *event,  in perf_event_task_output()  argument
8144 	if (!perf_event_task_match(event))  in perf_event_task_output()
8147 	perf_event_header__init_id(&task_event->event_id.header, &sample, event);  in perf_event_task_output()
8149 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_task_output()
8154 	task_event->event_id.pid = perf_event_pid(event, task);  in perf_event_task_output()
8155 	task_event->event_id.tid = perf_event_tid(event, task);  in perf_event_task_output()
8158 		task_event->event_id.ppid = perf_event_pid(event,  in perf_event_task_output()
8160 		task_event->event_id.ptid = perf_event_pid(event,  in perf_event_task_output()
8163 		task_event->event_id.ppid = perf_event_pid(event, current);  in perf_event_task_output()
8164 		task_event->event_id.ptid = perf_event_tid(event, current);  in perf_event_task_output()
8167 	task_event->event_id.time = perf_event_clock(event);  in perf_event_task_output()
8171 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_task_output()
8234 static int perf_event_comm_match(struct perf_event *event)  in perf_event_comm_match()  argument
8236 	return event->attr.comm;  in perf_event_comm_match()
8239 static void perf_event_comm_output(struct perf_event *event,  in perf_event_comm_output()  argument
8248 	if (!perf_event_comm_match(event))  in perf_event_comm_output()
8251 	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);  in perf_event_comm_output()
8252 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_comm_output()
8258 	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);  in perf_event_comm_output()
8259 	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);  in perf_event_comm_output()
8265 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_comm_output()
8333 static int perf_event_namespaces_match(struct perf_event *event)  in perf_event_namespaces_match()  argument
8335 	return event->attr.namespaces;  in perf_event_namespaces_match()
8338 static void perf_event_namespaces_output(struct perf_event *event,  in perf_event_namespaces_output()  argument
8347 	if (!perf_event_namespaces_match(event))  in perf_event_namespaces_output()
8351 				   &sample, event);  in perf_event_namespaces_output()
8352 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_namespaces_output()
8357 	namespaces_event->event_id.pid = perf_event_pid(event,  in perf_event_namespaces_output()
8359 	namespaces_event->event_id.tid = perf_event_tid(event,  in perf_event_namespaces_output()
8364 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_namespaces_output()
8461 static int perf_event_cgroup_match(struct perf_event *event)  in perf_event_cgroup_match()  argument
8463 	return event->attr.cgroup;  in perf_event_cgroup_match()
8466 static void perf_event_cgroup_output(struct perf_event *event, void *data)  in perf_event_cgroup_output()  argument
8474 	if (!perf_event_cgroup_match(event))  in perf_event_cgroup_output()
8478 				   &sample, event);  in perf_event_cgroup_output()
8479 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_cgroup_output()
8487 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_cgroup_output()
8572 static int perf_event_mmap_match(struct perf_event *event,  in perf_event_mmap_match()  argument
8579 	return (!executable && event->attr.mmap_data) ||  in perf_event_mmap_match()
8580 	       (executable && (event->attr.mmap || event->attr.mmap2));  in perf_event_mmap_match()
8583 static void perf_event_mmap_output(struct perf_event *event,  in perf_event_mmap_output()  argument
8594 	if (!perf_event_mmap_match(event, data))  in perf_event_mmap_output()
8597 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8607 	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);  in perf_event_mmap_output()
8608 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_mmap_output()
8613 	mmap_event->event_id.pid = perf_event_pid(event, current);  in perf_event_mmap_output()
8614 	mmap_event->event_id.tid = perf_event_tid(event, current);  in perf_event_mmap_output()
8616 	use_build_id = event->attr.build_id && mmap_event->build_id_size;  in perf_event_mmap_output()
8618 	if (event->attr.mmap2 && use_build_id)  in perf_event_mmap_output()
8623 	if (event->attr.mmap2) {  in perf_event_mmap_output()
8642 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_mmap_output()
8803 static void __perf_addr_filters_adjust(struct perf_event *event, void *data)  in __perf_addr_filters_adjust()  argument
8805 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in __perf_addr_filters_adjust()
8811 	if (!has_addr_filter(event))  in __perf_addr_filters_adjust()
8820 						&event->addr_filter_ranges[count]))  in __perf_addr_filters_adjust()
8827 		event->addr_filters_gen++;  in __perf_addr_filters_adjust()
8831 		perf_event_stop(event, 1);  in __perf_addr_filters_adjust()
8890 void perf_event_aux_event(struct perf_event *event, unsigned long head,  in perf_event_aux_event()  argument
8912 	perf_event_header__init_id(&rec.header, &sample, event);  in perf_event_aux_event()
8913 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);  in perf_event_aux_event()
8919 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_aux_event()
8927 void perf_log_lost_samples(struct perf_event *event, u64 lost)  in perf_log_lost_samples()  argument
8945 	perf_event_header__init_id(&lost_samples_event.header, &sample, event);  in perf_log_lost_samples()
8947 	ret = perf_output_begin(&handle, &sample, event,  in perf_log_lost_samples()
8953 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_lost_samples()
8972 static int perf_event_switch_match(struct perf_event *event)  in perf_event_switch_match()  argument
8974 	return event->attr.context_switch;  in perf_event_switch_match()
8977 static void perf_event_switch_output(struct perf_event *event, void *data)  in perf_event_switch_output()  argument
8984 	if (!perf_event_switch_match(event))  in perf_event_switch_output()
8988 	if (event->ctx->task) {  in perf_event_switch_output()
8995 					perf_event_pid(event, se->next_prev);  in perf_event_switch_output()
8997 					perf_event_tid(event, se->next_prev);  in perf_event_switch_output()
9000 	perf_event_header__init_id(&se->event_id.header, &sample, event);  in perf_event_switch_output()
9002 	ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size);  in perf_event_switch_output()
9006 	if (event->ctx->task)  in perf_event_switch_output()
9011 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_switch_output()
9049 static void perf_log_throttle(struct perf_event *event, int enable)  in perf_log_throttle()  argument
9066 		.time		= perf_event_clock(event),  in perf_log_throttle()
9067 		.id		= primary_event_id(event),  in perf_log_throttle()
9068 		.stream_id	= event->id,  in perf_log_throttle()
9074 	perf_event_header__init_id(&throttle_event.header, &sample, event);  in perf_log_throttle()
9076 	ret = perf_output_begin(&handle, &sample, event,  in perf_log_throttle()
9082 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_throttle()
9102 static int perf_event_ksymbol_match(struct perf_event *event)  in perf_event_ksymbol_match()  argument
9104 	return event->attr.ksymbol;  in perf_event_ksymbol_match()
9107 static void perf_event_ksymbol_output(struct perf_event *event, void *data)  in perf_event_ksymbol_output()  argument
9114 	if (!perf_event_ksymbol_match(event))  in perf_event_ksymbol_output()
9118 				   &sample, event);  in perf_event_ksymbol_output()
9119 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_ksymbol_output()
9126 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_ksymbol_output()
9192 static int perf_event_bpf_match(struct perf_event *event)  in perf_event_bpf_match()  argument
9194 	return event->attr.bpf_event;  in perf_event_bpf_match()
9197 static void perf_event_bpf_output(struct perf_event *event, void *data)  in perf_event_bpf_output()  argument
9204 	if (!perf_event_bpf_match(event))  in perf_event_bpf_output()
9208 				   &sample, event);  in perf_event_bpf_output()
9209 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_bpf_output()
9215 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_bpf_output()
9300 static int perf_event_text_poke_match(struct perf_event *event)  in perf_event_text_poke_match()  argument
9302 	return event->attr.text_poke;  in perf_event_text_poke_match()
9305 static void perf_event_text_poke_output(struct perf_event *event, void *data)  in perf_event_text_poke_output()  argument
9313 	if (!perf_event_text_poke_match(event))  in perf_event_text_poke_output()
9316 	perf_event_header__init_id(&text_poke_event->event_id.header, &sample, event);  in perf_event_text_poke_output()
9318 	ret = perf_output_begin(&handle, &sample, event,  in perf_event_text_poke_output()
9333 	perf_event__output_id_sample(event, &handle, &sample);  in perf_event_text_poke_output()
9370 void perf_event_itrace_started(struct perf_event *event)  in perf_event_itrace_started()  argument
9372 	event->attach_state |= PERF_ATTACH_ITRACE;  in perf_event_itrace_started()
9375 static void perf_log_itrace_start(struct perf_event *event)  in perf_log_itrace_start()  argument
9386 	if (event->parent)  in perf_log_itrace_start()
9387 		event = event->parent;  in perf_log_itrace_start()
9389 	if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) ||  in perf_log_itrace_start()
9390 	    event->attach_state & PERF_ATTACH_ITRACE)  in perf_log_itrace_start()
9396 	rec.pid	= perf_event_pid(event, current);  in perf_log_itrace_start()
9397 	rec.tid	= perf_event_tid(event, current);  in perf_log_itrace_start()
9399 	perf_event_header__init_id(&rec.header, &sample, event);  in perf_log_itrace_start()
9400 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);  in perf_log_itrace_start()
9406 	perf_event__output_id_sample(event, &handle, &sample);  in perf_log_itrace_start()
9411 void perf_report_aux_output_id(struct perf_event *event, u64 hw_id)  in perf_report_aux_output_id()  argument
9421 	if (event->parent)  in perf_report_aux_output_id()
9422 		event = event->parent;  in perf_report_aux_output_id()
9429 	perf_event_header__init_id(&rec.header, &sample, event);  in perf_report_aux_output_id()
9430 	ret = perf_output_begin(&handle, &sample, event, rec.header.size);  in perf_report_aux_output_id()
9436 	perf_event__output_id_sample(event, &handle, &sample);  in perf_report_aux_output_id()
9443 __perf_event_account_interrupt(struct perf_event *event, int throttle)  in __perf_event_account_interrupt()  argument
9445 	struct hw_perf_event *hwc = &event->hw;  in __perf_event_account_interrupt()
9460 			perf_log_throttle(event, 0);  in __perf_event_account_interrupt()
9465 	if (event->attr.freq) {  in __perf_event_account_interrupt()
9472 			perf_adjust_period(event, delta, hwc->last_period, true);  in __perf_event_account_interrupt()
9478 int perf_event_account_interrupt(struct perf_event *event)  in perf_event_account_interrupt()  argument
9480 	return __perf_event_account_interrupt(event, 1);  in perf_event_account_interrupt()
9483 static inline bool sample_is_allowed(struct perf_event *event, struct pt_regs *regs)  in sample_is_allowed()  argument
9490 	if (event->attr.exclude_kernel && !user_mode(regs))  in sample_is_allowed()
9500 static int __perf_event_overflow(struct perf_event *event,  in __perf_event_overflow()  argument
9504 	int events = atomic_read(&event->event_limit);  in __perf_event_overflow()
9511 	if (unlikely(!is_sampling_event(event)))  in __perf_event_overflow()
9514 	ret = __perf_event_account_interrupt(event, throttle);  in __perf_event_overflow()
9521 	event->pending_kill = POLL_IN;  in __perf_event_overflow()
9522 	if (events && atomic_dec_and_test(&event->event_limit)) {  in __perf_event_overflow()
9524 		event->pending_kill = POLL_HUP;  in __perf_event_overflow()
9525 		perf_event_disable_inatomic(event);  in __perf_event_overflow()
9528 	if (event->attr.sigtrap) {  in __perf_event_overflow()
9535 		bool valid_sample = sample_is_allowed(event, regs);  in __perf_event_overflow()
9540 		if (!event->pending_sigtrap) {  in __perf_event_overflow()
9541 			event->pending_sigtrap = pending_id;  in __perf_event_overflow()
9542 			local_inc(&event->ctx->nr_pending);  in __perf_event_overflow()
9543 		} else if (event->attr.exclude_kernel && valid_sample) {  in __perf_event_overflow()
9556 			WARN_ON_ONCE(event->pending_sigtrap != pending_id);  in __perf_event_overflow()
9559 		event->pending_addr = 0;  in __perf_event_overflow()
9561 			event->pending_addr = data->addr;  in __perf_event_overflow()
9562 		irq_work_queue(&event->pending_irq);  in __perf_event_overflow()
9565 	READ_ONCE(event->overflow_handler)(event, data, regs);  in __perf_event_overflow()  local
9567 	if (*perf_event_fasync(event) && event->pending_kill) {  in __perf_event_overflow()
9568 		event->pending_wakeup = 1;  in __perf_event_overflow()
9569 		irq_work_queue(&event->pending_irq);  in __perf_event_overflow()
9575 int perf_event_overflow(struct perf_event *event,  in perf_event_overflow()  argument
9579 	return __perf_event_overflow(event, 1, data, regs);  in perf_event_overflow()
9604 u64 perf_swevent_set_period(struct perf_event *event)  in perf_swevent_set_period()  argument
9606 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_set_period()
9627 static void perf_swevent_overflow(struct perf_event *event, u64 overflow,  in perf_swevent_overflow()  argument
9631 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_overflow()
9635 		overflow = perf_swevent_set_period(event);  in perf_swevent_overflow()
9641 		if (__perf_event_overflow(event, throttle,  in perf_swevent_overflow()
9653 static void perf_swevent_event(struct perf_event *event, u64 nr,  in perf_swevent_event()  argument
9657 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_event()
9659 	local64_add(nr, &event->count);  in perf_swevent_event()
9664 	if (!is_sampling_event(event))  in perf_swevent_event()
9667 	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {  in perf_swevent_event()
9669 		return perf_swevent_overflow(event, 1, data, regs);  in perf_swevent_event()
9671 		data->period = event->hw.last_period;  in perf_swevent_event()
9673 	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)  in perf_swevent_event()
9674 		return perf_swevent_overflow(event, 1, data, regs);  in perf_swevent_event()
9679 	perf_swevent_overflow(event, 0, data, regs);  in perf_swevent_event()
9682 static int perf_exclude_event(struct perf_event *event,  in perf_exclude_event()  argument
9685 	if (event->hw.state & PERF_HES_STOPPED)  in perf_exclude_event()
9689 		if (event->attr.exclude_user && user_mode(regs))  in perf_exclude_event()
9692 		if (event->attr.exclude_kernel && !user_mode(regs))  in perf_exclude_event()
9699 static int perf_swevent_match(struct perf_event *event,  in perf_swevent_match()  argument
9705 	if (event->attr.type != type)  in perf_swevent_match()
9708 	if (event->attr.config != event_id)  in perf_swevent_match()
9711 	if (perf_exclude_event(event, regs))  in perf_swevent_match()
9747 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)  in find_swevent_head()  argument
9750 	u32 event_id = event->attr.config;  in find_swevent_head()
9751 	u64 type = event->attr.type;  in find_swevent_head()
9759 					  lockdep_is_held(&event->ctx->lock));  in find_swevent_head()
9772 	struct perf_event *event;  in do_perf_sw_event()  local
9780 	hlist_for_each_entry_rcu(event, head, hlist_entry) {  in do_perf_sw_event()
9781 		if (perf_swevent_match(event, type, event_id, data, regs))  in do_perf_sw_event()
9782 			perf_swevent_event(event, nr, data, regs);  in do_perf_sw_event()
9832 static void perf_swevent_read(struct perf_event *event)  in perf_swevent_read()  argument
9836 static int perf_swevent_add(struct perf_event *event, int flags)  in perf_swevent_add()  argument
9839 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_add()
9842 	if (is_sampling_event(event)) {  in perf_swevent_add()
9844 		perf_swevent_set_period(event);  in perf_swevent_add()
9849 	head = find_swevent_head(swhash, event);  in perf_swevent_add()
9853 	hlist_add_head_rcu(&event->hlist_entry, head);  in perf_swevent_add()
9854 	perf_event_update_userpage(event);  in perf_swevent_add()
9859 static void perf_swevent_del(struct perf_event *event, int flags)  in perf_swevent_del()  argument
9861 	hlist_del_rcu(&event->hlist_entry);  in perf_swevent_del()
9864 static void perf_swevent_start(struct perf_event *event, int flags)  in perf_swevent_start()  argument
9866 	event->hw.state = 0;  in perf_swevent_start()
9869 static void perf_swevent_stop(struct perf_event *event, int flags)  in perf_swevent_stop()  argument
9871 	event->hw.state = PERF_HES_STOPPED;  in perf_swevent_stop()
9963 static void sw_perf_event_destroy(struct perf_event *event)  in sw_perf_event_destroy()  argument
9965 	u64 event_id = event->attr.config;  in sw_perf_event_destroy()
9967 	WARN_ON(event->parent);  in sw_perf_event_destroy()
9976 static int perf_swevent_init(struct perf_event *event)  in perf_swevent_init()  argument
9978 	u64 event_id = event->attr.config;  in perf_swevent_init()
9980 	if (event->attr.type != PERF_TYPE_SOFTWARE)  in perf_swevent_init()
9986 	if (has_branch_stack(event))  in perf_swevent_init()
9991 		event->attr.type = perf_cpu_clock.type;  in perf_swevent_init()
9994 		event->attr.type = perf_task_clock.type;  in perf_swevent_init()
10004 	if (!event->parent) {  in perf_swevent_init()
10012 		event->destroy = sw_perf_event_destroy;  in perf_swevent_init()
10033 static void tp_perf_event_destroy(struct perf_event *event)  in tp_perf_event_destroy()  argument
10035 	perf_trace_destroy(event);  in tp_perf_event_destroy()
10038 static int perf_tp_event_init(struct perf_event *event)  in perf_tp_event_init()  argument
10042 	if (event->attr.type != PERF_TYPE_TRACEPOINT)  in perf_tp_event_init()
10048 	if (has_branch_stack(event))  in perf_tp_event_init()
10051 	err = perf_trace_init(event);  in perf_tp_event_init()
10055 	event->destroy = tp_perf_event_destroy;  in perf_tp_event_init()
10071 static int perf_tp_filter_match(struct perf_event *event,  in perf_tp_filter_match()  argument
10077 	if (event->parent)  in perf_tp_filter_match()
10078 		event = event->parent;  in perf_tp_filter_match()
10080 	if (likely(!event->filter) || filter_match_preds(event->filter, record))  in perf_tp_filter_match()
10085 static int perf_tp_event_match(struct perf_event *event,  in perf_tp_event_match()  argument
10089 	if (event->hw.state & PERF_HES_STOPPED)  in perf_tp_event_match()
10094 	if (event->attr.exclude_kernel && !user_mode(regs))  in perf_tp_event_match()
10097 	if (!perf_tp_filter_match(event, data))  in perf_tp_event_match()
10115 	perf_tp_event(call->event.type, count, raw_data, size, regs, head,  in perf_trace_run_bpf_submit()
10123 					struct perf_event *event)  in __perf_tp_event_target_task()  argument
10127 	if (event->attr.config != entry->type)  in __perf_tp_event_target_task()
10130 	if (event->attr.sigtrap)  in __perf_tp_event_target_task()
10132 	if (perf_tp_event_match(event, data, regs))  in __perf_tp_event_target_task()
10133 		perf_swevent_event(event, count, data, regs);  in __perf_tp_event_target_task()
10143 	struct perf_event *event, *sibling;  in perf_tp_event_target_task()  local
10145 	perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) {  in perf_tp_event_target_task()
10146 		__perf_tp_event_target_task(count, record, regs, data, event);  in perf_tp_event_target_task()
10147 		for_each_sibling_event(sibling, event)  in perf_tp_event_target_task()
10151 	perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) {  in perf_tp_event_target_task()
10152 		__perf_tp_event_target_task(count, record, regs, data, event);  in perf_tp_event_target_task()
10153 		for_each_sibling_event(sibling, event)  in perf_tp_event_target_task()
10163 	struct perf_event *event;  in perf_tp_event()  local
10177 	hlist_for_each_entry_rcu(event, head, hlist_entry) {  in perf_tp_event()
10178 		if (perf_tp_event_match(event, &data, regs)) {  in perf_tp_event()
10179 			perf_swevent_event(event, count, &data, regs);  in perf_tp_event()
10257 static int perf_kprobe_event_init(struct perf_event *event);
10269 static int perf_kprobe_event_init(struct perf_event *event)  in perf_kprobe_event_init()  argument
10274 	if (event->attr.type != perf_kprobe.type)  in perf_kprobe_event_init()
10283 	if (has_branch_stack(event))  in perf_kprobe_event_init()
10286 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_kprobe_event_init()
10287 	err = perf_kprobe_init(event, is_retprobe);  in perf_kprobe_event_init()
10291 	event->destroy = perf_kprobe_destroy;  in perf_kprobe_event_init()
10316 static int perf_uprobe_event_init(struct perf_event *event);
10328 static int perf_uprobe_event_init(struct perf_event *event)  in perf_uprobe_event_init()  argument
10334 	if (event->attr.type != perf_uprobe.type)  in perf_uprobe_event_init()
10343 	if (has_branch_stack(event))  in perf_uprobe_event_init()
10346 	is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE;  in perf_uprobe_event_init()
10347 	ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT;  in perf_uprobe_event_init()
10348 	err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe);  in perf_uprobe_event_init()
10352 	event->destroy = perf_uprobe_destroy;  in perf_uprobe_event_init()
10369 static void perf_event_free_filter(struct perf_event *event)  in perf_event_free_filter()  argument
10371 	ftrace_profile_free_filter(event);  in perf_event_free_filter()
10375 static void bpf_overflow_handler(struct perf_event *event,  in bpf_overflow_handler()  argument
10381 		.event = event,  in bpf_overflow_handler()
10390 	prog = READ_ONCE(event->prog);  in bpf_overflow_handler()
10392 		perf_prepare_sample(data, event, regs);  in bpf_overflow_handler()
10401 	event->orig_overflow_handler(event, data, regs);  in bpf_overflow_handler()
10404 static int perf_event_set_bpf_handler(struct perf_event *event,  in perf_event_set_bpf_handler()  argument
10408 	if (event->overflow_handler_context)  in perf_event_set_bpf_handler()
10412 	if (event->prog)  in perf_event_set_bpf_handler()
10418 	if (event->attr.precise_ip &&  in perf_event_set_bpf_handler()
10420 	    (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||  in perf_event_set_bpf_handler()
10421 	     event->attr.exclude_callchain_kernel ||  in perf_event_set_bpf_handler()
10422 	     event->attr.exclude_callchain_user)) {  in perf_event_set_bpf_handler()
10435 	event->prog = prog;  in perf_event_set_bpf_handler()
10436 	event->bpf_cookie = bpf_cookie;  in perf_event_set_bpf_handler()
10437 	event->orig_overflow_handler = READ_ONCE(event->overflow_handler);  in perf_event_set_bpf_handler()
10438 	WRITE_ONCE(event->overflow_handler, bpf_overflow_handler);  in perf_event_set_bpf_handler()
10442 static void perf_event_free_bpf_handler(struct perf_event *event)  in perf_event_free_bpf_handler()  argument
10444 	struct bpf_prog *prog = event->prog;  in perf_event_free_bpf_handler()
10449 	WRITE_ONCE(event->overflow_handler, event->orig_overflow_handler);  in perf_event_free_bpf_handler()
10450 	event->prog = NULL;  in perf_event_free_bpf_handler()
10454 static int perf_event_set_bpf_handler(struct perf_event *event,  in perf_event_set_bpf_handler()  argument
10460 static void perf_event_free_bpf_handler(struct perf_event *event)  in perf_event_free_bpf_handler()  argument
10469 static inline bool perf_event_is_tracing(struct perf_event *event)  in perf_event_is_tracing()  argument
10471 	if (event->pmu == &perf_tracepoint)  in perf_event_is_tracing()
10474 	if (event->pmu == &perf_kprobe)  in perf_event_is_tracing()
10478 	if (event->pmu == &perf_uprobe)  in perf_event_is_tracing()
10484 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,  in perf_event_set_bpf_prog()  argument
10489 	if (!perf_event_is_tracing(event))  in perf_event_set_bpf_prog()
10490 		return perf_event_set_bpf_handler(event, prog, bpf_cookie);  in perf_event_set_bpf_prog()
10492 	is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE;  in perf_event_set_bpf_prog()
10493 	is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE;  in perf_event_set_bpf_prog()
10494 	is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT;  in perf_event_set_bpf_prog()
10495 	is_syscall_tp = is_syscall_trace_event(event->tp_event);  in perf_event_set_bpf_prog()
10514 		int off = trace_event_get_offsets(event->tp_event);  in perf_event_set_bpf_prog()
10520 	return perf_event_attach_bpf_prog(event, prog, bpf_cookie);  in perf_event_set_bpf_prog()
10523 void perf_event_free_bpf_prog(struct perf_event *event)  in perf_event_free_bpf_prog()  argument
10525 	if (!perf_event_is_tracing(event)) {  in perf_event_free_bpf_prog()
10526 		perf_event_free_bpf_handler(event);  in perf_event_free_bpf_prog()
10529 	perf_event_detach_bpf_prog(event);  in perf_event_free_bpf_prog()
10538 static void perf_event_free_filter(struct perf_event *event)  in perf_event_free_filter()  argument
10542 int perf_event_set_bpf_prog(struct perf_event *event, struct bpf_prog *prog,  in perf_event_set_bpf_prog()  argument
10548 void perf_event_free_bpf_prog(struct perf_event *event)  in perf_event_free_bpf_prog()  argument
10570 perf_addr_filter_new(struct perf_event *event, struct list_head *filters)  in perf_addr_filter_new()  argument
10572 	int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu);  in perf_addr_filter_new()
10599 static void perf_addr_filters_splice(struct perf_event *event,  in perf_addr_filters_splice()  argument
10605 	if (!has_addr_filter(event))  in perf_addr_filters_splice()
10609 	if (event->parent)  in perf_addr_filters_splice()
10612 	raw_spin_lock_irqsave(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10614 	list_splice_init(&event->addr_filters.list, &list);  in perf_addr_filters_splice()
10616 		list_splice(head, &event->addr_filters.list);  in perf_addr_filters_splice()
10618 	raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags);  in perf_addr_filters_splice()
10648 static void perf_event_addr_filters_apply(struct perf_event *event)  in perf_event_addr_filters_apply()  argument
10650 	struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_addr_filters_apply()
10651 	struct task_struct *task = READ_ONCE(event->ctx->task);  in perf_event_addr_filters_apply()
10679 			event->addr_filter_ranges[count].start = 0;  in perf_event_addr_filters_apply()
10680 			event->addr_filter_ranges[count].size = 0;  in perf_event_addr_filters_apply()
10682 			perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]);  in perf_event_addr_filters_apply()
10684 			event->addr_filter_ranges[count].start = filter->offset;  in perf_event_addr_filters_apply()
10685 			event->addr_filter_ranges[count].size  = filter->size;  in perf_event_addr_filters_apply()
10691 	event->addr_filters_gen++;  in perf_event_addr_filters_apply()
10701 	perf_event_stop(event, 1);  in perf_event_addr_filters_apply()
10755 perf_event_parse_addr_filter(struct perf_event *event, char *fstr,  in perf_event_parse_addr_filter()  argument
10782 			filter = perf_addr_filter_new(event, filters);  in perf_event_parse_addr_filter()
10868 				if (!event->ctx->task)  in perf_event_parse_addr_filter()
10883 				event->addr_filters.nr_file_filters++;  in perf_event_parse_addr_filter()
10912 perf_event_set_addr_filter(struct perf_event *event, char *filter_str)  in perf_event_set_addr_filter()  argument
10921 	lockdep_assert_held(&event->ctx->mutex);  in perf_event_set_addr_filter()
10923 	if (WARN_ON_ONCE(event->parent))  in perf_event_set_addr_filter()
10926 	ret = perf_event_parse_addr_filter(event, filter_str, &filters);  in perf_event_set_addr_filter()
10930 	ret = event->pmu->addr_filters_validate(&filters);  in perf_event_set_addr_filter()
10935 	perf_addr_filters_splice(event, &filters);  in perf_event_set_addr_filter()
10938 	perf_event_for_each_child(event, perf_event_addr_filters_apply);  in perf_event_set_addr_filter()
10946 	event->addr_filters.nr_file_filters = 0;  in perf_event_set_addr_filter()
10951 static int perf_event_set_filter(struct perf_event *event, void __user *arg)  in perf_event_set_filter()  argument
10961 	if (perf_event_is_tracing(event)) {  in perf_event_set_filter()
10962 		struct perf_event_context *ctx = event->ctx;  in perf_event_set_filter()
10976 		ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);  in perf_event_set_filter()
10980 	if (has_addr_filter(event))  in perf_event_set_filter()
10981 		ret = perf_event_set_addr_filter(event, filter_str);  in perf_event_set_filter()
10996 	struct perf_event *event;  in perf_swevent_hrtimer()  local
10999 	event = container_of(hrtimer, struct perf_event, hw.hrtimer);  in perf_swevent_hrtimer()
11001 	if (event->state != PERF_EVENT_STATE_ACTIVE)  in perf_swevent_hrtimer()
11004 	event->pmu->read(event);  in perf_swevent_hrtimer()
11006 	perf_sample_data_init(&data, 0, event->hw.last_period);  in perf_swevent_hrtimer()
11009 	if (regs && !perf_exclude_event(event, regs)) {  in perf_swevent_hrtimer()
11010 		if (!(event->attr.exclude_idle && is_idle_task(current)))  in perf_swevent_hrtimer()
11011 			if (__perf_event_overflow(event, 1, &data, regs))  in perf_swevent_hrtimer()
11015 	period = max_t(u64, 10000, event->hw.sample_period);  in perf_swevent_hrtimer()
11021 static void perf_swevent_start_hrtimer(struct perf_event *event)  in perf_swevent_start_hrtimer()  argument
11023 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_start_hrtimer()
11026 	if (!is_sampling_event(event))  in perf_swevent_start_hrtimer()
11042 static void perf_swevent_cancel_hrtimer(struct perf_event *event)  in perf_swevent_cancel_hrtimer()  argument
11044 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_cancel_hrtimer()
11046 	if (is_sampling_event(event)) {  in perf_swevent_cancel_hrtimer()
11054 static void perf_swevent_init_hrtimer(struct perf_event *event)  in perf_swevent_init_hrtimer()  argument
11056 	struct hw_perf_event *hwc = &event->hw;  in perf_swevent_init_hrtimer()
11058 	if (!is_sampling_event(event))  in perf_swevent_init_hrtimer()
11068 	if (event->attr.freq) {  in perf_swevent_init_hrtimer()
11069 		long freq = event->attr.sample_freq;  in perf_swevent_init_hrtimer()
11071 		event->attr.sample_period = NSEC_PER_SEC / freq;  in perf_swevent_init_hrtimer()
11072 		hwc->sample_period = event->attr.sample_period;  in perf_swevent_init_hrtimer()
11075 		event->attr.freq = 0;  in perf_swevent_init_hrtimer()
11083 static void cpu_clock_event_update(struct perf_event *event)  in cpu_clock_event_update()  argument
11089 	prev = local64_xchg(&event->hw.prev_count, now);  in cpu_clock_event_update()
11090 	local64_add(now - prev, &event->count);  in cpu_clock_event_update()
11093 static void cpu_clock_event_start(struct perf_event *event, int flags)  in cpu_clock_event_start()  argument
11095 	local64_set(&event->hw.prev_count, local_clock());  in cpu_clock_event_start()
11096 	perf_swevent_start_hrtimer(event);  in cpu_clock_event_start()
11099 static void cpu_clock_event_stop(struct perf_event *event, int flags)  in cpu_clock_event_stop()  argument
11101 	perf_swevent_cancel_hrtimer(event);  in cpu_clock_event_stop()
11102 	cpu_clock_event_update(event);  in cpu_clock_event_stop()
11105 static int cpu_clock_event_add(struct perf_event *event, int flags)  in cpu_clock_event_add()  argument
11108 		cpu_clock_event_start(event, flags);  in cpu_clock_event_add()
11109 	perf_event_update_userpage(event);  in cpu_clock_event_add()
11114 static void cpu_clock_event_del(struct perf_event *event, int flags)  in cpu_clock_event_del()  argument
11116 	cpu_clock_event_stop(event, flags);  in cpu_clock_event_del()
11119 static void cpu_clock_event_read(struct perf_event *event)  in cpu_clock_event_read()  argument
11121 	cpu_clock_event_update(event);  in cpu_clock_event_read()
11124 static int cpu_clock_event_init(struct perf_event *event)  in cpu_clock_event_init()  argument
11126 	if (event->attr.type != perf_cpu_clock.type)  in cpu_clock_event_init()
11129 	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)  in cpu_clock_event_init()
11135 	if (has_branch_stack(event))  in cpu_clock_event_init()
11138 	perf_swevent_init_hrtimer(event);  in cpu_clock_event_init()
11161 static void task_clock_event_update(struct perf_event *event, u64 now)  in task_clock_event_update()  argument
11166 	prev = local64_xchg(&event->hw.prev_count, now);  in task_clock_event_update()
11168 	local64_add(delta, &event->count);  in task_clock_event_update()
11171 static void task_clock_event_start(struct perf_event *event, int flags)  in task_clock_event_start()  argument
11173 	local64_set(&event->hw.prev_count, event->ctx->time);  in task_clock_event_start()
11174 	perf_swevent_start_hrtimer(event);  in task_clock_event_start()
11177 static void task_clock_event_stop(struct perf_event *event, int flags)  in task_clock_event_stop()  argument
11179 	perf_swevent_cancel_hrtimer(event);  in task_clock_event_stop()
11180 	task_clock_event_update(event, event->ctx->time);  in task_clock_event_stop()
11183 static int task_clock_event_add(struct perf_event *event, int flags)  in task_clock_event_add()  argument
11186 		task_clock_event_start(event, flags);  in task_clock_event_add()
11187 	perf_event_update_userpage(event);  in task_clock_event_add()
11192 static void task_clock_event_del(struct perf_event *event, int flags)  in task_clock_event_del()  argument
11194 	task_clock_event_stop(event, PERF_EF_UPDATE);  in task_clock_event_del()
11197 static void task_clock_event_read(struct perf_event *event)  in task_clock_event_read()  argument
11200 	u64 delta = now - event->ctx->timestamp;  in task_clock_event_read()
11201 	u64 time = event->ctx->time + delta;  in task_clock_event_read()
11203 	task_clock_event_update(event, time);  in task_clock_event_read()
11206 static int task_clock_event_init(struct perf_event *event)  in task_clock_event_init()  argument
11208 	if (event->attr.type != perf_task_clock.type)  in task_clock_event_init()
11211 	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)  in task_clock_event_init()
11217 	if (has_branch_stack(event))  in task_clock_event_init()
11220 	perf_swevent_init_hrtimer(event);  in task_clock_event_init()
11252 static int perf_event_nop_int(struct perf_event *event, u64 value)  in perf_event_nop_int()  argument
11294 static int perf_event_idx_default(struct perf_event *event)  in perf_event_idx_default()  argument
11572 static inline bool has_extended_regs(struct perf_event *event)  in has_extended_regs()  argument
11574 	return (event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK) ||  in has_extended_regs()
11575 	       (event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK);  in has_extended_regs()
11578 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event)  in perf_try_init_event()  argument
11592 	if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) {  in perf_try_init_event()
11597 		ctx = perf_event_ctx_lock_nested(event->group_leader,  in perf_try_init_event()
11602 	event->pmu = pmu;  in perf_try_init_event()
11603 	ret = pmu->event_init(event);  in perf_try_init_event()
11606 		perf_event_ctx_unlock(event->group_leader, ctx);  in perf_try_init_event()
11610 		    has_extended_regs(event))  in perf_try_init_event()
11614 		    event_has_any_exclude_flag(event))  in perf_try_init_event()
11617 		if (ret && event->destroy)  in perf_try_init_event()
11618 			event->destroy(event);  in perf_try_init_event()
11627 static struct pmu *perf_init_event(struct perf_event *event)  in perf_init_event()  argument
11639 	event->orig_type = event->attr.type;  in perf_init_event()
11642 	if (event->parent && event->parent->pmu) {  in perf_init_event()
11643 		pmu = event->parent->pmu;  in perf_init_event()
11644 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
11653 	type = event->attr.type;  in perf_init_event()
11655 		type = event->attr.config >> PERF_PMU_TYPE_SHIFT;  in perf_init_event()
11660 			event->attr.config &= PERF_HW_EVENT_MASK;  in perf_init_event()
11669 		if (event->attr.type != type && type != PERF_TYPE_RAW &&  in perf_init_event()
11673 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
11674 		if (ret == -ENOENT && event->attr.type != type && !extended_type) {  in perf_init_event()
11675 			type = event->attr.type;  in perf_init_event()
11686 		ret = perf_try_init_event(pmu, event);  in perf_init_event()
11703 static void attach_sb_event(struct perf_event *event)  in attach_sb_event()  argument
11705 	struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu);  in attach_sb_event()
11708 	list_add_rcu(&event->sb_list, &pel->list);  in attach_sb_event()
11719 static void account_pmu_sb_event(struct perf_event *event)  in account_pmu_sb_event()  argument
11721 	if (is_sb_event(event))  in account_pmu_sb_event()
11722 		attach_sb_event(event);  in account_pmu_sb_event()
11746 static void account_event(struct perf_event *event)  in account_event()  argument
11750 	if (event->parent)  in account_event()
11753 	if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB))  in account_event()
11755 	if (event->attr.mmap || event->attr.mmap_data)  in account_event()
11757 	if (event->attr.build_id)  in account_event()
11759 	if (event->attr.comm)  in account_event()
11761 	if (event->attr.namespaces)  in account_event()
11763 	if (event->attr.cgroup)  in account_event()
11765 	if (event->attr.task)  in account_event()
11767 	if (event->attr.freq)  in account_event()
11769 	if (event->attr.context_switch) {  in account_event()
11773 	if (has_branch_stack(event))  in account_event()
11775 	if (is_cgroup_event(event))  in account_event()
11777 	if (event->attr.ksymbol)  in account_event()
11779 	if (event->attr.bpf_event)  in account_event()
11781 	if (event->attr.text_poke)  in account_event()
11812 	account_pmu_sb_event(event);  in account_event()
11827 	struct perf_event *event;  in perf_event_alloc()  local
11842 	event = kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO,  in perf_event_alloc()
11844 	if (!event)  in perf_event_alloc()
11852 		group_leader = event;  in perf_event_alloc()
11854 	mutex_init(&event->child_mutex);  in perf_event_alloc()
11855 	INIT_LIST_HEAD(&event->child_list);  in perf_event_alloc()
11857 	INIT_LIST_HEAD(&event->event_entry);  in perf_event_alloc()
11858 	INIT_LIST_HEAD(&event->sibling_list);  in perf_event_alloc()
11859 	INIT_LIST_HEAD(&event->active_list);  in perf_event_alloc()
11860 	init_event_group(event);  in perf_event_alloc()
11861 	INIT_LIST_HEAD(&event->rb_entry);  in perf_event_alloc()
11862 	INIT_LIST_HEAD(&event->active_entry);  in perf_event_alloc()
11863 	INIT_LIST_HEAD(&event->addr_filters.list);  in perf_event_alloc()
11864 	INIT_HLIST_NODE(&event->hlist_entry);  in perf_event_alloc()
11867 	init_waitqueue_head(&event->waitq);  in perf_event_alloc()
11868 	init_irq_work(&event->pending_irq, perf_pending_irq);  in perf_event_alloc()
11869 	init_task_work(&event->pending_task, perf_pending_task);  in perf_event_alloc()
11871 	mutex_init(&event->mmap_mutex);  in perf_event_alloc()
11872 	raw_spin_lock_init(&event->addr_filters.lock);  in perf_event_alloc()
11874 	atomic_long_set(&event->refcount, 1);  in perf_event_alloc()
11875 	event->cpu		= cpu;  in perf_event_alloc()
11876 	event->attr		= *attr;  in perf_event_alloc()
11877 	event->group_leader	= group_leader;  in perf_event_alloc()
11878 	event->pmu		= NULL;  in perf_event_alloc()
11879 	event->oncpu		= -1;  in perf_event_alloc()
11881 	event->parent		= parent_event;  in perf_event_alloc()
11883 	event->ns		= get_pid_ns(task_active_pid_ns(current));  in perf_event_alloc()
11884 	event->id		= atomic64_inc_return(&perf_event_id);  in perf_event_alloc()
11886 	event->state		= PERF_EVENT_STATE_INACTIVE;  in perf_event_alloc()
11889 		event->event_caps = parent_event->event_caps;  in perf_event_alloc()
11892 		event->attach_state = PERF_ATTACH_TASK;  in perf_event_alloc()
11898 		event->hw.target = get_task_struct(task);  in perf_event_alloc()
11901 	event->clock = &local_clock;  in perf_event_alloc()
11903 		event->clock = parent_event->clock;  in perf_event_alloc()
11913 			event->prog = prog;  in perf_event_alloc()
11914 			event->orig_overflow_handler =  in perf_event_alloc()
11921 		event->overflow_handler	= overflow_handler;  in perf_event_alloc()
11922 		event->overflow_handler_context = context;  in perf_event_alloc()
11923 	} else if (is_write_backward(event)){  in perf_event_alloc()
11924 		event->overflow_handler = perf_event_output_backward;  in perf_event_alloc()
11925 		event->overflow_handler_context = NULL;  in perf_event_alloc()
11927 		event->overflow_handler = perf_event_output_forward;  in perf_event_alloc()
11928 		event->overflow_handler_context = NULL;  in perf_event_alloc()
11931 	perf_event__state_init(event);  in perf_event_alloc()
11935 	hwc = &event->hw;  in perf_event_alloc()
11950 	if (!has_branch_stack(event))  in perf_event_alloc()
11951 		event->attr.branch_sample_type = 0;  in perf_event_alloc()
11953 	pmu = perf_init_event(event);  in perf_event_alloc()
11969 	if (event->attr.aux_output &&  in perf_event_alloc()
11976 		err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader);  in perf_event_alloc()
11981 	err = exclusive_event_init(event);  in perf_event_alloc()
11985 	if (has_addr_filter(event)) {  in perf_event_alloc()
11986 		event->addr_filter_ranges = kcalloc(pmu->nr_addr_filters,  in perf_event_alloc()
11989 		if (!event->addr_filter_ranges) {  in perf_event_alloc()
11998 		if (event->parent) {  in perf_event_alloc()
11999 			struct perf_addr_filters_head *ifh = perf_event_addr_filters(event);  in perf_event_alloc()
12002 			memcpy(event->addr_filter_ranges,  in perf_event_alloc()
12003 			       event->parent->addr_filter_ranges,  in perf_event_alloc()
12009 		event->addr_filters_gen = 1;  in perf_event_alloc()
12012 	if (!event->parent) {  in perf_event_alloc()
12013 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {  in perf_event_alloc()
12020 	err = security_perf_event_alloc(event);  in perf_event_alloc()
12025 	account_event(event);  in perf_event_alloc()
12027 	return event;  in perf_event_alloc()
12030 	if (!event->parent) {  in perf_event_alloc()
12031 		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)  in perf_event_alloc()
12035 	kfree(event->addr_filter_ranges);  in perf_event_alloc()
12038 	exclusive_event_destroy(event);  in perf_event_alloc()
12041 	if (is_cgroup_event(event))  in perf_event_alloc()
12042 		perf_detach_cgroup(event);  in perf_event_alloc()
12043 	if (event->destroy)  in perf_event_alloc()
12044 		event->destroy(event);  in perf_event_alloc()
12047 	if (event->hw.target)  in perf_event_alloc()
12048 		put_task_struct(event->hw.target);  in perf_event_alloc()
12049 	call_rcu(&event->rcu_head, free_event_rcu);  in perf_event_alloc()
12190 perf_event_set_output(struct perf_event *event, struct perf_event *output_event)  in perf_event_set_output()  argument
12196 		mutex_lock(&event->mmap_mutex);  in perf_event_set_output()
12201 	if (event == output_event)  in perf_event_set_output()
12207 	if (output_event->cpu != event->cpu)  in perf_event_set_output()
12213 	if (output_event->cpu == -1 && output_event->hw.target != event->hw.target)  in perf_event_set_output()
12219 	if (output_event->clock != event->clock)  in perf_event_set_output()
12226 	if (is_write_backward(output_event) != is_write_backward(event))  in perf_event_set_output()
12232 	if (has_aux(event) && has_aux(output_event) &&  in perf_event_set_output()
12233 	    event->pmu != output_event->pmu)  in perf_event_set_output()
12243 	mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex);  in perf_event_set_output()
12246 	if (atomic_read(&event->mmap_count))  in perf_event_set_output()
12262 	ring_buffer_attach(event, rb);  in perf_event_set_output()
12266 	mutex_unlock(&event->mmap_mutex);  in perf_event_set_output()
12274 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id)  in perf_event_set_clock()  argument
12280 		event->clock = &ktime_get_mono_fast_ns;  in perf_event_set_clock()
12285 		event->clock = &ktime_get_raw_fast_ns;  in perf_event_set_clock()
12290 		event->clock = &ktime_get_real_ns;  in perf_event_set_clock()
12294 		event->clock = &ktime_get_boottime_ns;  in perf_event_set_clock()
12298 		event->clock = &ktime_get_clocktai_ns;  in perf_event_set_clock()
12305 	if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI))  in perf_event_set_clock()
12357 	struct perf_event *event, *sibling;  in SYSCALL_DEFINE5()  local
12460 	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,  in SYSCALL_DEFINE5()
12462 	if (IS_ERR(event)) {  in SYSCALL_DEFINE5()
12463 		err = PTR_ERR(event);  in SYSCALL_DEFINE5()
12467 	if (is_sampling_event(event)) {  in SYSCALL_DEFINE5()
12468 		if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {  in SYSCALL_DEFINE5()
12478 	pmu = event->pmu;  in SYSCALL_DEFINE5()
12481 		err = perf_event_set_clock(event, attr.clockid);  in SYSCALL_DEFINE5()
12487 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in SYSCALL_DEFINE5()
12508 	ctx = find_get_context(task, event);  in SYSCALL_DEFINE5()
12528 		struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu);  in SYSCALL_DEFINE5()
12547 		if (group_leader->clock != event->clock)  in SYSCALL_DEFINE5()
12555 		if (group_leader->cpu != event->cpu)  in SYSCALL_DEFINE5()
12570 		if (is_software_event(event) &&  in SYSCALL_DEFINE5()
12584 		} else if (!is_software_event(event)) {  in SYSCALL_DEFINE5()
12605 	pmu_ctx = find_get_pmu_context(pmu, ctx, event);  in SYSCALL_DEFINE5()
12610 	event->pmu_ctx = pmu_ctx;  in SYSCALL_DEFINE5()
12613 		err = perf_event_set_output(event, output_event);  in SYSCALL_DEFINE5()
12618 	if (!perf_event_validate_size(event)) {  in SYSCALL_DEFINE5()
12623 	if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) {  in SYSCALL_DEFINE5()
12632 	if (!exclusive_event_installable(event, ctx)) {  in SYSCALL_DEFINE5()
12639 	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags);  in SYSCALL_DEFINE5()
12694 	perf_event__header_size(event);  in SYSCALL_DEFINE5()
12695 	perf_event__id_header_size(event);  in SYSCALL_DEFINE5()
12697 	event->owner = current;  in SYSCALL_DEFINE5()
12699 	perf_install_in_context(ctx, event, event->cpu);  in SYSCALL_DEFINE5()
12710 	list_add_tail(&event->owner_entry, &current->perf_event_list);  in SYSCALL_DEFINE5()
12724 	put_pmu_ctx(event->pmu_ctx);  in SYSCALL_DEFINE5()
12725 	event->pmu_ctx = NULL; /* _free_event() */  in SYSCALL_DEFINE5()
12734 	free_event(event);  in SYSCALL_DEFINE5()
12762 	struct perf_event *event;  in perf_event_create_kernel_counter()  local
12773 	event = perf_event_alloc(attr, cpu, task, NULL, NULL,  in perf_event_create_kernel_counter()
12775 	if (IS_ERR(event)) {  in perf_event_create_kernel_counter()
12776 		err = PTR_ERR(event);  in perf_event_create_kernel_counter()
12781 	event->owner = TASK_TOMBSTONE;  in perf_event_create_kernel_counter()
12782 	pmu = event->pmu;  in perf_event_create_kernel_counter()
12785 		event->event_caps |= PERF_EV_CAP_SOFTWARE;  in perf_event_create_kernel_counter()
12790 	ctx = find_get_context(task, event);  in perf_event_create_kernel_counter()
12803 	pmu_ctx = find_get_pmu_context(pmu, ctx, event);  in perf_event_create_kernel_counter()
12808 	event->pmu_ctx = pmu_ctx;  in perf_event_create_kernel_counter()
12825 	if (!exclusive_event_installable(event, ctx)) {  in perf_event_create_kernel_counter()
12830 	perf_install_in_context(ctx, event, event->cpu);  in perf_event_create_kernel_counter()
12834 	return event;  in perf_event_create_kernel_counter()
12838 	event->pmu_ctx = NULL; /* _free_event() */  in perf_event_create_kernel_counter()
12844 	free_event(event);  in perf_event_create_kernel_counter()
12855 	struct perf_event *event, *sibling;  in __perf_pmu_remove()  local
12857 	perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) {  in __perf_pmu_remove()
12858 		perf_remove_from_context(event, 0);  in __perf_pmu_remove()
12859 		put_pmu_ctx(event->pmu_ctx);  in __perf_pmu_remove()
12860 		list_add(&event->migrate_entry, events);  in __perf_pmu_remove()
12862 		for_each_sibling_event(sibling, event) {  in __perf_pmu_remove()
12872 				     int cpu, struct perf_event *event)  in __perf_pmu_install_event()  argument
12876 	event->cpu = cpu;  in __perf_pmu_install_event()
12877 	epc = find_get_pmu_context(pmu, ctx, event);  in __perf_pmu_install_event()
12878 	event->pmu_ctx = epc;  in __perf_pmu_install_event()
12880 	if (event->state >= PERF_EVENT_STATE_OFF)  in __perf_pmu_install_event()
12881 		event->state = PERF_EVENT_STATE_INACTIVE;  in __perf_pmu_install_event()
12882 	perf_install_in_context(ctx, event, cpu);  in __perf_pmu_install_event()
12888 	struct perf_event *event, *tmp;  in __perf_pmu_install()  local
12898 	list_for_each_entry_safe(event, tmp, events, migrate_entry) {  in __perf_pmu_install()
12899 		if (event->group_leader == event)  in __perf_pmu_install()
12902 		list_del(&event->migrate_entry);  in __perf_pmu_install()
12903 		__perf_pmu_install_event(pmu, ctx, cpu, event);  in __perf_pmu_install()
12910 	list_for_each_entry_safe(event, tmp, events, migrate_entry) {  in __perf_pmu_install()
12911 		list_del(&event->migrate_entry);  in __perf_pmu_install()
12912 		__perf_pmu_install_event(pmu, ctx, cpu, event);  in __perf_pmu_install()
12972 perf_event_exit_event(struct perf_event *event, struct perf_event_context *ctx)  in perf_event_exit_event()  argument
12974 	struct perf_event *parent_event = event->parent;  in perf_event_exit_event()
12994 	perf_remove_from_context(event, detach_flags);  in perf_event_exit_event()
12997 	if (event->state > PERF_EVENT_STATE_EXIT)  in perf_event_exit_event()
12998 		perf_event_set_state(event, PERF_EVENT_STATE_EXIT);  in perf_event_exit_event()
13010 		free_event(event);  in perf_event_exit_event()
13018 	perf_event_wakeup(event);  in perf_event_exit_event()
13090 	struct perf_event *event, *tmp;  in perf_event_exit_task()  local
13093 	list_for_each_entry_safe(event, tmp, &child->perf_event_list,  in perf_event_exit_task()
13095 		list_del_init(&event->owner_entry);  in perf_event_exit_task()
13102 		smp_store_release(&event->owner, NULL);  in perf_event_exit_task()
13117 static void perf_free_event(struct perf_event *event,  in perf_free_event()  argument
13120 	struct perf_event *parent = event->parent;  in perf_free_event()
13126 	list_del_init(&event->child_list);  in perf_free_event()
13132 	perf_group_detach(event);  in perf_free_event()
13133 	list_del_event(event, ctx);  in perf_free_event()
13135 	free_event(event);  in perf_free_event()
13148 	struct perf_event *event, *tmp;  in perf_event_free_task()  local
13168 	list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry)  in perf_event_free_task()
13169 		perf_free_event(event, ctx);  in perf_event_free_task()
13218 const struct perf_event_attr *perf_event_attrs(struct perf_event *event)  in perf_event_attrs()  argument
13220 	if (!event)  in perf_event_attrs()
13223 	return &event->attr;  in perf_event_attrs()
13392 inherit_task_group(struct perf_event *event, struct task_struct *parent,  in inherit_task_group()  argument
13400 	if (!event->attr.inherit ||  in inherit_task_group()
13401 	    (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) ||  in inherit_task_group()
13403 	    (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) {  in inherit_task_group()
13423 	ret = inherit_group(event, parent, parent_ctx, child, child_ctx);  in inherit_task_group()
13437 	struct perf_event *event;  in perf_event_init_context()  local
13471 	perf_event_groups_for_each(event, &parent_ctx->pinned_groups) {  in perf_event_init_context()
13472 		ret = inherit_task_group(event, parent, parent_ctx,  in perf_event_init_context()
13487 	perf_event_groups_for_each(event, &parent_ctx->flexible_groups) {  in perf_event_init_context()
13488 		ret = inherit_task_group(event, parent, parent_ctx,  in perf_event_init_context()
13595 	struct perf_event *event;  in __perf_event_exit_context()  local
13599 	list_for_each_entry(event, &ctx->event_list, event_entry)  in __perf_event_exit_context()
13600 		__perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP);  in __perf_event_exit_context()