4  *    Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
5  *    Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
6  *    Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
12  * For licencing details see kernel-base/COPYING
21  * Kernel-internal data types and definitions:
39 #include <linux/rhashtable-types.h>
100 	return frag->pad < sizeof(u64);  in perf_raw_frag_last()
108  *          -1ULL means invalid/unknown.
118  * The hw_idx index is between -1 (unknown) and max depth,
138 	int		idx;	/* index in shared_regs->regs[] */
144  * PERF_EVENT_FLAG_ARCH bits are reserved for architecture-specific
153  * struct hw_perf_event - performance event hardware details:
175 			/* for tp_event->class */
220 #define PERF_HES_UPTODATE	0x02 /* event->count up-to-date */
300  * struct pmu - generic performance monitoring unit
314 	 * various common per-pmu feature flags
338 	 *  -ENOENT	-- @event is not for this PMU
340 	 *  -ENODEV	-- @event is for this PMU but PMU not present
341 	 *  -EBUSY	-- @event is for this PMU but PMU temporarily unavailable
342 	 *  -EINVAL	-- @event is for this PMU but @event is not valid
343 	 *  -EOPNOTSUPP -- @event is for this PMU, @event is valid, but not supported
344 	 *  -EACCES	-- @event is for this PMU, @event is valid, but no privileges
346 	 *  0		-- @event is for this PMU and valid
360 	 * Flags for ->add()/->del()/ ->start()/->stop(). There are
369 	 * transaction, see the ->*_txn() methods.
378 	 * ->add() called without PERF_EF_START should result in the same state
379 	 *  as ->add() followed by ->stop().
381 	 * ->del() must always PERF_EF_UPDATE stop an event. If it calls
382 	 *  ->stop() that must deal with already being stopped without
392 	 * returns !0. ->start() will be used to continue.
397 	 * is on -- will be called from NMI context with the PMU generates
400 	 * ->stop() with PERF_EF_UPDATE will read the counter and update
401 	 *  period/count values like ->read() would.
403 	 * ->start() with PERF_EF_RELOAD will reprogram the counter
404 	 *  value, must be preceded by a ->stop() with PERF_EF_UPDATE.
422 	 * Start the transaction, after this ->add() doesn't need to
429 	 * If ->start_txn() disabled the ->add() schedulability test
430 	 * then ->commit_txn() is required to perform one. On success
432 	 * open until ->cancel_txn() is called.
438 	 * Will cancel the transaction, assumes ->del() is called
439 	 * for each successful ->add() during the transaction.
453 	 * context-switches callback
464 	 * PMU specific parts of task perf event context (i.e. ctx->task_ctx_data)
474 	 * Set up pmu-private data structures for an AUX area
481 	 * Free pmu-private AUX data structures
487 	 * state, so that preempting ->start()/->stop() callbacks does
501 	 * supplied filters are valid, -errno otherwise.
511 	 * translate hw-agnostic filters into hardware configuration in
514 	 * Runs as a part of filter sync sequence that is done in ->start()
528 	 * or non-zero for "match".
552  * struct perf_addr_filter - address range filter definition
554  * @path:	object file's path for file-based filters
559  * This is a hardware-agnostic filter configuration as specified by the user.
570  * struct perf_addr_filters_head - container for address range filters
574  * @nr_file_filters:	number of file-based filters
591  * enum perf_event_state - the states of an event:
594 	PERF_EVENT_STATE_DEAD		= -4,
595 	PERF_EVENT_STATE_EXIT		= -3,
596 	PERF_EVENT_STATE_ERROR		= -2,
597 	PERF_EVENT_STATE_OFF		= -1,
649  * event->sibling_list is modified whole holding both ctx->lock and ctx->mutex
650  * as such iteration must hold either lock. However, since ctx->lock is an IRQ
652  * disabled is sufficient since it will hold-off the IPIs.
658 		      lockdep_is_held(&(event)->ctx->mutex) != LOCK_STATE_HELD))
665 	if ((event)->group_leader == (event))			\
666 		list_for_each_entry((sibling), &(event)->sibling_list, sibling_list)
669  * struct perf_event - performance event kernel representation:
675 	 *   modifications require ctx->lock
681 	 * Locked for modification by both ctx->mutex and ctx->lock; holding
710 	 * event->pmu will always point to pmu in which this event belongs.
711 	 * Whereas event->pmu_ctx->pmu may point to other pmu when group of
725 	 * been scheduled in, if this is a per-task event)
740 	 * event->pmu_ctx points to perf_event_pmu_context in which the event
742 	 * sw event is part of a group which also contains non-sw events.
794 	/* vma address array for file-based filders */
836 	 * Certain events gets forwarded to another pmu internally by over-
837 	 * writing kernel copy of event->attr.type without user being aware
838 	 * of it. event->orig_type contains original 'type' requested by
846  *           ,-----------------------[1:n]----------------------.
848  * perf_event_context <-[1:n]-> perf_event_pmu_context <--- perf_event
850  *           `--------[1:n]---------'     `-[n:1]-> pmu <-[1:n]-'
857  *   modification, both: ctx->mutex && ctx->lock
858  *   reading, either:    ctx->mutex || ctx->lock
861  * with ctx->mutex held; this means that as long as we can guarantee the epc
862  * has events the above rules hold.
865  * ctx->mutex pinning the configuration. Since we hold a reference on
867  * associated pmu_ctx must exist and cannot change due to ctx->mutex.
878 	/* Used to avoid freeing per-cpu perf_event_pmu_context */
883 	atomic_t			refcount; /* event <-> epc */
903  * struct perf_event_context - event context structure
905  * Used as a container for task events and CPU events as well:
934 	refcount_t			refcount; /* event <-> ctx */
958 	 * Sum (event->pending_sigtrap + event->pending_work)
960 	 * The SIGTRAP is targeted at ctx->task, as such it won't do changing
989  * struct perf_event_cpu_context - per cpu event context structure
1001 	 * Per-CPU storage for iterators used in visit_groups_merge. The default
1032  * This is a per-cpu dynamically allocated data structure.
1055 					   ctx ? lockdep_is_held(&ctx->lock)  in perf_cgroup_from_task()
1119 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS;  in branch_sample_no_flags()
1124 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES;  in branch_sample_no_cycles()
1129 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_TYPE_SAVE;  in branch_sample_type()
1134 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;  in branch_sample_hw_index()
1139 	return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;  in branch_sample_priv()
1205 	data->sample_flags = PERF_SAMPLE_PERIOD;  in perf_sample_data_init()
1206 	data->period = period;  in perf_sample_data_init()
1207 	data->dyn_size = 0;  in perf_sample_data_init()
1210 		data->addr = addr;  in perf_sample_data_init()
1211 		data->sample_flags |= PERF_SAMPLE_ADDR;  in perf_sample_data_init()
1221 	data->callchain = perf_callchain(event, regs);  in perf_sample_save_callchain()
1222 	size += data->callchain->nr;  in perf_sample_save_callchain()
1224 	data->dyn_size += size * sizeof(u64);  in perf_sample_save_callchain()
1225 	data->sample_flags |= PERF_SAMPLE_CALLCHAIN;  in perf_sample_save_callchain()
1231 	struct perf_raw_frag *frag = &raw->frag;  in perf_sample_save_raw_data()
1236 		sum += frag->size;  in perf_sample_save_raw_data()
1239 		frag = frag->next;  in perf_sample_save_raw_data()
1243 	raw->size = size - sizeof(u32);  in perf_sample_save_raw_data()
1244 	frag->pad = raw->size - sum;  in perf_sample_save_raw_data()
1246 	data->raw = raw;  in perf_sample_save_raw_data()
1247 	data->dyn_size += size;  in perf_sample_save_raw_data()
1248 	data->sample_flags |= PERF_SAMPLE_RAW;  in perf_sample_save_raw_data()
1259 	size += brs->nr * sizeof(struct perf_branch_entry);  in perf_sample_save_brstack()
1261 	data->br_stack = brs;  in perf_sample_save_brstack()
1262 	data->dyn_size += size;  in perf_sample_save_brstack()
1263 	data->sample_flags |= PERF_SAMPLE_BRANCH_STACK;  in perf_sample_save_brstack()
1271 	size += event->header_size + event->id_header_size;  in perf_sample_data_size()
1272 	size += data->dyn_size;  in perf_sample_data_size()
1284 	br->mispred = 0;  in perf_clear_branch_entry_bitfields()
1285 	br->predicted = 0;  in perf_clear_branch_entry_bitfields()
1286 	br->in_tx = 0;  in perf_clear_branch_entry_bitfields()
1287 	br->abort = 0;  in perf_clear_branch_entry_bitfields()
1288 	br->cycles = 0;  in perf_clear_branch_entry_bitfields()
1289 	br->type = 0;  in perf_clear_branch_entry_bitfields()
1290 	br->spec = PERF_BR_SPEC_NA;  in perf_clear_branch_entry_bitfields()
1291 	br->reserved = 0;  in perf_clear_branch_entry_bitfields()
1331 	__is_default_overflow_handler((event)->overflow_handler)
1339 	return __is_default_overflow_handler(event->orig_overflow_handler);  in uses_default_overflow_handler()
1360 	struct perf_event_attr *attr = &event->attr;  in event_has_any_exclude_flag()
1362 	return attr->exclude_idle || attr->exclude_user ||  in event_has_any_exclude_flag()
1363 	       attr->exclude_kernel || attr->exclude_hv ||  in event_has_any_exclude_flag()
1364 	       attr->exclude_guest || attr->exclude_host;  in event_has_any_exclude_flag()
1369 	return event->attr.sample_period != 0;  in is_sampling_event()
1377 	return event->event_caps & PERF_EV_CAP_SOFTWARE;  in is_software_event()
1385 	return event->pmu_ctx->pmu->task_ctx_nr == perf_sw_context;  in in_software_context()
1390 	return pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE;  in is_exclusive_pmu()
1403  * When generating a perf sample in-line, instead of from an interrupt /
1405  * like: SW_CONTEXT_SWITCHES, SW_MIGRATIONS and the tie-in with tracepoints.
1408  * - ip for PERF_SAMPLE_IP
1409  * - cs for user_mode() tests
1410  * - sp for PERF_SAMPLE_CALLCHAIN
1411  * - eflags for MISC bits and CALLCHAIN (see: perf_hw_regs())
1453 		task->sched_migrated = 1;  in perf_event_task_migrate()
1463 	    task->sched_migrated) {  in perf_event_task_sched_in()
1465 		task->sched_migrated = 0;  in perf_event_task_sched_in()
1497 DECLARE_STATIC_CALL(__perf_guest_state, *perf_guest_cbs->state);
1498 DECLARE_STATIC_CALL(__perf_guest_get_ip, *perf_guest_cbs->get_ip);
1499 DECLARE_STATIC_CALL(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr);
1547 	if (ctx->contexts < sysctl_perf_event_max_contexts_per_stack) {  in perf_callchain_store_context()
1548 		struct perf_callchain_entry *entry = ctx->entry;  in perf_callchain_store_context()
1549 		entry->ip[entry->nr++] = ip;  in perf_callchain_store_context()
1550 		++ctx->contexts;  in perf_callchain_store_context()
1553 		ctx->contexts_maxed = true;  in perf_callchain_store_context()
1554 		return -1; /* no more room, stop walking the stack */  in perf_callchain_store_context()
1560 	if (ctx->nr < ctx->max_stack && !ctx->contexts_maxed) {  in perf_callchain_store()
1561 		struct perf_callchain_entry *entry = ctx->entry;  in perf_callchain_store()
1562 		entry->ip[entry->nr++] = ip;  in perf_callchain_store()
1563 		++ctx->nr;  in perf_callchain_store()
1566 		return -1; /* no more room, stop walking the stack */  in perf_callchain_store()
1594 	return sysctl_perf_event_paranoid > -1;  in perf_is_paranoid()
1600 		return -EACCES;  in perf_allow_kernel()
1608 		return -EACCES;  in perf_allow_cpu()
1615 	if (sysctl_perf_event_paranoid > -1 && !perfmon_capable())  in perf_allow_tracepoint()
1616 		return -EPERM;  in perf_allow_tracepoint()
1639 	return event->attr.sample_type & PERF_SAMPLE_BRANCH_STACK;  in has_branch_stack()
1644 	return event->attr.branch_sample_type != 0;  in needs_branch_stack()
1649 	return event->pmu->setup_aux;  in has_aux()
1654 	return !!event->attr.write_backward;  in is_write_backward()
1659 	return event->pmu->nr_addr_filters;  in has_addr_filter()
1668 	struct perf_addr_filters_head *ifh = &event->addr_filters;  in perf_event_addr_filters()
1670 	if (event->parent)  in perf_event_addr_filters()
1671 		ifh = &event->parent->addr_filters;  in perf_event_addr_filters()
1719 		     unsigned long size)				{ return -EINVAL; }  in perf_aux_output_skip()
1735 static inline struct file *perf_event_get(unsigned int fd)	{ return ERR_PTR(-EINVAL); }  in perf_event_get()
1738 	return ERR_PTR(-EINVAL);  in perf_get_event()
1742 	return ERR_PTR(-EINVAL);  in perf_event_attrs()
1747 	return -EINVAL;  in perf_event_read_local()
1750 static inline int perf_event_task_disable(void)				{ return -EINVAL; }  in perf_event_task_disable()
1751 static inline int perf_event_task_enable(void)				{ return -EINVAL; }  in perf_event_task_enable()
1754 	return -EINVAL;  in perf_event_refresh()
1780 static inline int  perf_swevent_get_recursion_context(void)		{ return -1; }  in perf_swevent_get_recursion_context()
1785 static inline int __perf_event_disable(void *info)			{ return -1; }  in __perf_event_disable()
1790 	return -EINVAL;  in perf_event_period()