2  * Copyright © 2015-2016 Intel Corporation
40  * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
44  * without special privileges. Access to system-wide metrics requires root
58  * might sample sets of tightly-coupled counters, depending on the
70  * interleaved with event-type specific members.
74  * with the CPU, using HW specific packing formats for counter sets. Sometimes
76  * would be acceptable to expose them to unprivileged applications - to hide
88  * into perf's currently cpu centric design.
96  * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
102  * For posterity, in case we might re-visit trying to adapt core perf to be
106  * - The perf based OA PMU driver broke some significant design assumptions:
108  *   Existing perf pmus are used for profiling work on a cpu and we were
110  *   implications, the need to fake cpu-related data (such as user/kernel
112  *   as a way to forward device-specific status records.
115  *   involvement from the CPU, making our PMU driver the first of a kind.
117  *   Given the way we were periodically forward data from the GPU-mapped, OA
123  *   explicitly initiated from the cpu (say in response to a userspace read())
125  *   trigger a report from the cpu on demand.
130  *   opened, there's no clear precedent for being able to provide group-wide
139  *   for combining with the side-band raw reports it captures using
142  *   - As a side note on perf's grouping feature; there was also some concern
158  *   one time.  The OA unit is not designed to allow re-configuration while in
178  * - It felt like our perf based PMU was making some technical compromises
182  *   cpu core, while our device pmu related to neither.  Events opened with a
184  *   that process - so not appropriate for us. When an event is related to a
185  *   cpu id, perf ensures pmu methods will be invoked via an inter process
187  *   perf events for a specific cpu. This was workable but it meant the
223 #define OA_TAKEN(tail, head)	((tail - head) & (OA_BUFFER_SIZE - 1))
231  * CPU).
234  * by checking for a zeroed report-id field in tail reports, we want to account
254  * non-periodic reports (such as on context switch) or the OA unit may be
308  * code assumes all reports have a power-of-two size and ~(size - 1) can
328  * struct perf_open_properties - for validated properties given to open a stream
343  * @poll_oa_period: The period in nanoseconds at which the CPU will check for OA
387 	kfree(oa_config->flex_regs);  in i915_oa_config_release()
388 	kfree(oa_config->b_counter_regs);  in i915_oa_config_release()
389 	kfree(oa_config->mux_regs);  in i915_oa_config_release()
400 	oa_config = idr_find(&perf->metrics_idr, metrics_set);  in i915_perf_get_oa_config()
410 	i915_oa_config_put(oa_bo->oa_config);  in free_oa_config_bo()
411 	i915_vma_put(oa_bo->vma);  in free_oa_config_bo()
417 	struct intel_uncore *uncore = stream->uncore;  in gen12_oa_hw_tail_read()
425 	struct intel_uncore *uncore = stream->uncore;  in gen8_oa_hw_tail_read()
432 	struct intel_uncore *uncore = stream->uncore;  in gen7_oa_hw_tail_read()
439  * oa_buffer_check_unlocked - check for data and update tail ptr state
447  * pointer having a race with respect to what data is visible to the CPU.
464 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in oa_buffer_check_unlocked()
465 	int report_size = stream->oa_buffer.format_size;  in oa_buffer_check_unlocked()
475 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in oa_buffer_check_unlocked()
477 	hw_tail = stream->perf->ops.oa_hw_tail_read(stream);  in oa_buffer_check_unlocked()
482 	hw_tail &= ~(report_size - 1);  in oa_buffer_check_unlocked()
486 	if (hw_tail == stream->oa_buffer.aging_tail &&  in oa_buffer_check_unlocked()
487 	    (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) {  in oa_buffer_check_unlocked()
492 		stream->oa_buffer.tail = stream->oa_buffer.aging_tail;  in oa_buffer_check_unlocked()
498 		 * anywhere between this head and stream->oa_buffer.tail.  in oa_buffer_check_unlocked()
500 		head = stream->oa_buffer.head - gtt_offset;  in oa_buffer_check_unlocked()
501 		aged_tail = stream->oa_buffer.tail - gtt_offset;  in oa_buffer_check_unlocked()
503 		hw_tail -= gtt_offset;  in oa_buffer_check_unlocked()
518 			u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail);  in oa_buffer_check_unlocked()
523 			tail = (tail - report_size) & (OA_BUFFER_SIZE - 1);  in oa_buffer_check_unlocked()
527 		    __ratelimit(&stream->perf->tail_pointer_race))  in oa_buffer_check_unlocked()
532 		stream->oa_buffer.tail = gtt_offset + tail;  in oa_buffer_check_unlocked()
533 		stream->oa_buffer.aging_tail = gtt_offset + hw_tail;  in oa_buffer_check_unlocked()
534 		stream->oa_buffer.aging_timestamp = now;  in oa_buffer_check_unlocked()
537 	pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset,  in oa_buffer_check_unlocked()
538 			  stream->oa_buffer.head - gtt_offset) >= report_size;  in oa_buffer_check_unlocked()
540 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in oa_buffer_check_unlocked()
546  * append_oa_status - Appends a status record to a userspace read() buffer.
547  * @stream: An i915-perf stream opened for OA metrics
568 	if ((count - *offset) < header.size)  in append_oa_status()
569 		return -ENOSPC;  in append_oa_status()
572 		return -EFAULT;  in append_oa_status()
580  * append_oa_sample - Copies single OA report into userspace read() buffer.
581  * @stream: An i915-perf stream opened for OA metrics
588  * properties when opening a stream, tracked as `stream->sample_flags`. This
602 	int report_size = stream->oa_buffer.format_size;  in append_oa_sample()
607 	header.size = stream->sample_size;  in append_oa_sample()
609 	if ((count - *offset) < header.size)  in append_oa_sample()
610 		return -ENOSPC;  in append_oa_sample()
614 		return -EFAULT;  in append_oa_sample()
618 		return -EFAULT;  in append_oa_sample()
626  * gen8_append_oa_reports - Copies all buffered OA reports into
628  * @stream: An i915-perf stream opened for OA metrics
633  * Notably any error condition resulting in a short read (-%ENOSPC or
634  * -%EFAULT) will be returned even though one or more records may
641  * and back-to-front you're not alone, but this follows the
651 	struct intel_uncore *uncore = stream->uncore;  in gen8_append_oa_reports()
652 	int report_size = stream->oa_buffer.format_size;  in gen8_append_oa_reports()
653 	u8 *oa_buf_base = stream->oa_buffer.vaddr;  in gen8_append_oa_reports()
654 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in gen8_append_oa_reports()
655 	u32 mask = (OA_BUFFER_SIZE - 1);  in gen8_append_oa_reports()
662 	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))  in gen8_append_oa_reports()
663 		return -EIO;  in gen8_append_oa_reports()
665 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen8_append_oa_reports()
667 	head = stream->oa_buffer.head;  in gen8_append_oa_reports()
668 	tail = stream->oa_buffer.tail;  in gen8_append_oa_reports()
670 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen8_append_oa_reports()
676 	head -= gtt_offset;  in gen8_append_oa_reports()
677 	tail -= gtt_offset;  in gen8_append_oa_reports()
686 	if (drm_WARN_ONCE(&uncore->i915->drm,  in gen8_append_oa_reports()
691 		return -EIO;  in gen8_append_oa_reports()
711 		if (drm_WARN_ON(&uncore->i915->drm,  in gen8_append_oa_reports()
712 				(OA_BUFFER_SIZE - head) < report_size)) {  in gen8_append_oa_reports()
713 			drm_err(&uncore->i915->drm,  in gen8_append_oa_reports()
714 				"Spurious OA head ptr: non-integral report offset\n");  in gen8_append_oa_reports()
728 			  (GRAPHICS_VER(stream->perf->i915) == 12 ?  in gen8_append_oa_reports()
732 		ctx_id = report32[2] & stream->specific_ctx_id_mask;  in gen8_append_oa_reports()
736 		 * invalid to be sure we avoid false-positive, single-context  in gen8_append_oa_reports()
742 		if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) &&  in gen8_append_oa_reports()
743 		    GRAPHICS_VER(stream->perf->i915) <= 11)  in gen8_append_oa_reports()
749 		 * stop the counters from updating as system-wide / global  in gen8_append_oa_reports()
753 		 * filtered on the cpu but it's not worth trying to  in gen8_append_oa_reports()
757 		 * provide a side-band view of the real values.  in gen8_append_oa_reports()
761 		 * needs be forwarded bookend context-switch reports so that it  in gen8_append_oa_reports()
774 		 * switches since it's not-uncommon for periodic samples to  in gen8_append_oa_reports()
777 		if (!stream->perf->exclusive_stream->ctx ||  in gen8_append_oa_reports()
778 		    stream->specific_ctx_id == ctx_id ||  in gen8_append_oa_reports()
779 		    stream->oa_buffer.last_ctx_id == stream->specific_ctx_id ||  in gen8_append_oa_reports()
786 			if (stream->perf->exclusive_stream->ctx &&  in gen8_append_oa_reports()
787 			    stream->specific_ctx_id != ctx_id) {  in gen8_append_oa_reports()
796 			stream->oa_buffer.last_ctx_id = ctx_id;  in gen8_append_oa_reports()
810 		oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ?  in gen8_append_oa_reports()
813 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen8_append_oa_reports()
822 		stream->oa_buffer.head = head;  in gen8_append_oa_reports()
824 		spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen8_append_oa_reports()
831  * gen8_oa_read - copy status records then buffered OA reports
832  * @stream: An i915-perf stream opened for OA metrics
855 	struct intel_uncore *uncore = stream->uncore;  in gen8_oa_read()
860 	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))  in gen8_oa_read()
861 		return -EIO;  in gen8_oa_read()
863 	oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ?  in gen8_oa_read()
888 		drm_dbg(&stream->perf->i915->drm,  in gen8_oa_read()
890 			stream->period_exponent);  in gen8_oa_read()
892 		stream->perf->ops.oa_disable(stream);  in gen8_oa_read()
893 		stream->perf->ops.oa_enable(stream);  in gen8_oa_read()
896 		 * Note: .oa_enable() is expected to re-init the oabuffer and  in gen8_oa_read()
911 				 IS_GRAPHICS_VER(uncore->i915, 8, 11) ?  in gen8_oa_read()
920  * gen7_append_oa_reports - Copies all buffered OA reports into
922  * @stream: An i915-perf stream opened for OA metrics
927  * Notably any error condition resulting in a short read (-%ENOSPC or
928  * -%EFAULT) will be returned even though one or more records may
935  * and back-to-front you're not alone, but this follows the
945 	struct intel_uncore *uncore = stream->uncore;  in gen7_append_oa_reports()
946 	int report_size = stream->oa_buffer.format_size;  in gen7_append_oa_reports()
947 	u8 *oa_buf_base = stream->oa_buffer.vaddr;  in gen7_append_oa_reports()
948 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in gen7_append_oa_reports()
949 	u32 mask = (OA_BUFFER_SIZE - 1);  in gen7_append_oa_reports()
956 	if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled))  in gen7_append_oa_reports()
957 		return -EIO;  in gen7_append_oa_reports()
959 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen7_append_oa_reports()
961 	head = stream->oa_buffer.head;  in gen7_append_oa_reports()
962 	tail = stream->oa_buffer.tail;  in gen7_append_oa_reports()
964 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen7_append_oa_reports()
969 	head -= gtt_offset;  in gen7_append_oa_reports()
970 	tail -= gtt_offset;  in gen7_append_oa_reports()
978 	if (drm_WARN_ONCE(&uncore->i915->drm,  in gen7_append_oa_reports()
983 		return -EIO;  in gen7_append_oa_reports()
1000 		if (drm_WARN_ON(&uncore->i915->drm,  in gen7_append_oa_reports()
1001 				(OA_BUFFER_SIZE - head) < report_size)) {  in gen7_append_oa_reports()
1002 			drm_err(&uncore->i915->drm,  in gen7_append_oa_reports()
1003 				"Spurious OA head ptr: non-integral report offset\n");  in gen7_append_oa_reports()
1007 		/* The report-ID field for periodic samples includes  in gen7_append_oa_reports()
1014 			if (__ratelimit(&stream->perf->spurious_report_rs))  in gen7_append_oa_reports()
1031 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen7_append_oa_reports()
1041 		stream->oa_buffer.head = head;  in gen7_append_oa_reports()
1043 		spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen7_append_oa_reports()
1050  * gen7_oa_read - copy status records then buffered OA reports
1051  * @stream: An i915-perf stream opened for OA metrics
1070 	struct intel_uncore *uncore = stream->uncore;  in gen7_oa_read()
1074 	if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr))  in gen7_oa_read()
1075 		return -EIO;  in gen7_oa_read()
1084 	oastatus1 &= ~stream->perf->gen7_latched_oastatus1;  in gen7_oa_read()
1088 	 * - The status can be interpreted to mean that the buffer is  in gen7_oa_read()
1090 	 *   which will start to report a near-empty buffer after an  in gen7_oa_read()
1095 	 * - Since it also implies the HW has started overwriting old  in gen7_oa_read()
1100 	 * - In the future we may want to introduce a flight recorder  in gen7_oa_read()
1112 		drm_dbg(&stream->perf->i915->drm,  in gen7_oa_read()
1114 			stream->period_exponent);  in gen7_oa_read()
1116 		stream->perf->ops.oa_disable(stream);  in gen7_oa_read()
1117 		stream->perf->ops.oa_enable(stream);  in gen7_oa_read()
1127 		stream->perf->gen7_latched_oastatus1 |=  in gen7_oa_read()
1135  * i915_oa_wait_unlocked - handles blocking IO until OA data available
1136  * @stream: An i915-perf stream opened for OA metrics
1139  * for OA metrics. It waits until the hrtimer callback finds a non-empty
1143  * since any subsequent read handling will return -EAGAIN if there isn't
1151 	if (!stream->periodic)  in i915_oa_wait_unlocked()
1152 		return -EIO;  in i915_oa_wait_unlocked()
1154 	return wait_event_interruptible(stream->poll_wq,  in i915_oa_wait_unlocked()
1159  * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1160  * @stream: An i915-perf stream opened for OA metrics
1172 	poll_wait(file, &stream->poll_wq, wait);  in i915_oa_poll_wait()
1176  * i915_oa_read - just calls through to &i915_oa_ops->read
1177  * @stream: An i915-perf stream opened for OA metrics
1192 	return stream->perf->ops.read(stream, buf, count, offset);  in i915_oa_read()
1198 	struct i915_gem_context *ctx = stream->ctx;  in oa_pin_context()
1201 	int err = -ENODEV;  in oa_pin_context()
1204 		if (ce->engine != stream->engine) /* first match! */  in oa_pin_context()
1222 	if (err == -EDEADLK) {  in oa_pin_context()
1232 	stream->pinned_ctx = ce;  in oa_pin_context()
1233 	return stream->pinned_ctx;  in oa_pin_context()
1237  * oa_get_render_ctx_id - determine and hold ctx hw id
1238  * @stream: An i915-perf stream opened for OA metrics
1254 	switch (GRAPHICS_VER(ce->engine->i915)) {  in oa_get_render_ctx_id()
1260 		stream->specific_ctx_id = i915_ggtt_offset(ce->state);  in oa_get_render_ctx_id()
1261 		stream->specific_ctx_id_mask = 0;  in oa_get_render_ctx_id()
1267 		if (intel_engine_uses_guc(ce->engine)) {  in oa_get_render_ctx_id()
1278 			stream->specific_ctx_id = ce->lrc.lrca >> 12;  in oa_get_render_ctx_id()
1284 			stream->specific_ctx_id_mask =  in oa_get_render_ctx_id()
1285 				(1U << (GEN8_CTX_ID_WIDTH - 1)) - 1;  in oa_get_render_ctx_id()
1287 			stream->specific_ctx_id_mask =  in oa_get_render_ctx_id()
1288 				(1U << GEN8_CTX_ID_WIDTH) - 1;  in oa_get_render_ctx_id()
1289 			stream->specific_ctx_id = stream->specific_ctx_id_mask;  in oa_get_render_ctx_id()
1295 		if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 50)) {  in oa_get_render_ctx_id()
1296 			stream->specific_ctx_id_mask =  in oa_get_render_ctx_id()
1297 				((1U << XEHP_SW_CTX_ID_WIDTH) - 1) <<  in oa_get_render_ctx_id()
1298 				(XEHP_SW_CTX_ID_SHIFT - 32);  in oa_get_render_ctx_id()
1299 			stream->specific_ctx_id =  in oa_get_render_ctx_id()
1300 				(XEHP_MAX_CONTEXT_HW_ID - 1) <<  in oa_get_render_ctx_id()
1301 				(XEHP_SW_CTX_ID_SHIFT - 32);  in oa_get_render_ctx_id()
1303 			stream->specific_ctx_id_mask =  in oa_get_render_ctx_id()
1304 				((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32);  in oa_get_render_ctx_id()
1307 			 * 0 - BITS_PER_LONG are used by other contexts  in oa_get_render_ctx_id()
1310 			stream->specific_ctx_id =  in oa_get_render_ctx_id()
1311 				(GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32);  in oa_get_render_ctx_id()
1316 		MISSING_CASE(GRAPHICS_VER(ce->engine->i915));  in oa_get_render_ctx_id()
1319 	ce->tag = stream->specific_ctx_id;  in oa_get_render_ctx_id()
1321 	drm_dbg(&stream->perf->i915->drm,  in oa_get_render_ctx_id()
1323 		stream->specific_ctx_id,  in oa_get_render_ctx_id()
1324 		stream->specific_ctx_id_mask);  in oa_get_render_ctx_id()
1330  * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1331  * @stream: An i915-perf stream opened for OA metrics
1340 	ce = fetch_and_zero(&stream->pinned_ctx);  in oa_put_render_ctx_id()
1342 		ce->tag = 0; /* recomputed on next submission after parking */  in oa_put_render_ctx_id()
1346 	stream->specific_ctx_id = INVALID_CTX_ID;  in oa_put_render_ctx_id()
1347 	stream->specific_ctx_id_mask = 0;  in oa_put_render_ctx_id()
1353 	i915_vma_unpin_and_release(&stream->oa_buffer.vma,  in free_oa_buffer()
1356 	stream->oa_buffer.vaddr = NULL;  in free_oa_buffer()
1364 	i915_oa_config_put(stream->oa_config);  in free_oa_configs()
1365 	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)  in free_oa_configs()
1372 	i915_vma_unpin_and_release(&stream->noa_wait, 0);  in free_noa_wait()
1377 	struct i915_perf *perf = stream->perf;  in i915_oa_stream_destroy()
1379 	if (WARN_ON(stream != perf->exclusive_stream))  in i915_oa_stream_destroy()
1388 	WRITE_ONCE(perf->exclusive_stream, NULL);  in i915_oa_stream_destroy()
1389 	perf->ops.disable_metric_set(stream);  in i915_oa_stream_destroy()
1393 	intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);  in i915_oa_stream_destroy()
1394 	intel_engine_pm_put(stream->engine);  in i915_oa_stream_destroy()
1396 	if (stream->ctx)  in i915_oa_stream_destroy()
1402 	if (perf->spurious_report_rs.missed) {  in i915_oa_stream_destroy()
1404 			 perf->spurious_report_rs.missed);  in i915_oa_stream_destroy()
1410 	struct intel_uncore *uncore = stream->uncore;  in gen7_init_oa_buffer()
1411 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in gen7_init_oa_buffer()
1414 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen7_init_oa_buffer()
1416 	/* Pre-DevBDW: OABUFFER must be set with counters off,  in gen7_init_oa_buffer()
1421 	stream->oa_buffer.head = gtt_offset;  in gen7_init_oa_buffer()
1429 	stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;  in gen7_init_oa_buffer()
1430 	stream->oa_buffer.tail = gtt_offset;  in gen7_init_oa_buffer()
1432 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen7_init_oa_buffer()
1438 	stream->perf->gen7_latched_oastatus1 = 0;  in gen7_init_oa_buffer()
1442 	 * first allocating), we may re-init the OA buffer, either  in gen7_init_oa_buffer()
1443 	 * when re-enabling a stream or in error/reset paths.  in gen7_init_oa_buffer()
1445 	 * The reason we clear the buffer for each re-init is for the  in gen7_init_oa_buffer()
1447 	 * report-id field to make sure it's non-zero which relies on  in gen7_init_oa_buffer()
1451 	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);  in gen7_init_oa_buffer()
1456 	struct intel_uncore *uncore = stream->uncore;  in gen8_init_oa_buffer()
1457 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in gen8_init_oa_buffer()
1460 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen8_init_oa_buffer()
1464 	stream->oa_buffer.head = gtt_offset;  in gen8_init_oa_buffer()
1481 	stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;  in gen8_init_oa_buffer()
1482 	stream->oa_buffer.tail = gtt_offset;  in gen8_init_oa_buffer()
1489 	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;  in gen8_init_oa_buffer()
1491 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen8_init_oa_buffer()
1496 	 * first allocating), we may re-init the OA buffer, either  in gen8_init_oa_buffer()
1497 	 * when re-enabling a stream or in error/reset paths.  in gen8_init_oa_buffer()
1499 	 * The reason we clear the buffer for each re-init is for the  in gen8_init_oa_buffer()
1501 	 * reason field to make sure it's non-zero which relies on  in gen8_init_oa_buffer()
1505 	memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE);  in gen8_init_oa_buffer()
1510 	struct intel_uncore *uncore = stream->uncore;  in gen12_init_oa_buffer()
1511 	u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma);  in gen12_init_oa_buffer()
1514 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in gen12_init_oa_buffer()
1519 	stream->oa_buffer.head = gtt_offset;  in gen12_init_oa_buffer()
1535 	stream->oa_buffer.aging_tail = INVALID_TAIL_PTR;  in gen12_init_oa_buffer()
1536 	stream->oa_buffer.tail = gtt_offset;  in gen12_init_oa_buffer()
1543 	stream->oa_buffer.last_ctx_id = INVALID_CTX_ID;  in gen12_init_oa_buffer()
1545 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in gen12_init_oa_buffer()
1550 	 * first allocating), we may re-init the OA buffer, either  in gen12_init_oa_buffer()
1551 	 * when re-enabling a stream or in error/reset paths.  in gen12_init_oa_buffer()
1553 	 * The reason we clear the buffer for each re-init is for the  in gen12_init_oa_buffer()
1555 	 * reason field to make sure it's non-zero which relies on  in gen12_init_oa_buffer()
1559 	memset(stream->oa_buffer.vaddr, 0,  in gen12_init_oa_buffer()
1560 	       stream->oa_buffer.vma->size);  in gen12_init_oa_buffer()
1565 	struct drm_i915_private *i915 = stream->perf->i915;  in alloc_oa_buffer()
1570 	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma))  in alloc_oa_buffer()
1571 		return -ENODEV;  in alloc_oa_buffer()
1576 	bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE);  in alloc_oa_buffer()
1578 		drm_err(&i915->drm, "Failed to allocate OA buffer\n");  in alloc_oa_buffer()
1590 	stream->oa_buffer.vma = vma;  in alloc_oa_buffer()
1592 	stream->oa_buffer.vaddr =  in alloc_oa_buffer()
1594 	if (IS_ERR(stream->oa_buffer.vaddr)) {  in alloc_oa_buffer()
1595 		ret = PTR_ERR(stream->oa_buffer.vaddr);  in alloc_oa_buffer()
1607 	stream->oa_buffer.vaddr = NULL;  in alloc_oa_buffer()
1608 	stream->oa_buffer.vma = NULL;  in alloc_oa_buffer()
1622 	if (GRAPHICS_VER(stream->perf->i915) >= 8)  in save_restore_register()
1628 		*cs++ = intel_gt_scratch_offset(stream->engine->gt,  in save_restore_register()
1638 	struct drm_i915_private *i915 = stream->perf->i915;  in alloc_noa_wait()
1641 	const u64 delay_ticks = 0xffffffffffffffff -  in alloc_noa_wait()
1642 		intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915),  in alloc_noa_wait()
1643 		atomic64_read(&stream->perf->noa_programming_delay));  in alloc_noa_wait()
1644 	const u32 base = stream->engine->mmio_base;  in alloc_noa_wait()
1660 		drm_err(&i915->drm,  in alloc_noa_wait()
1708 	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);  in alloc_noa_wait()
1726 	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);  in alloc_noa_wait()
1746 	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);  in alloc_noa_wait()
1755 	*cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4;  in alloc_noa_wait()
1760 	 *      (((1 * << 64) - 1) - delay_ns)  in alloc_noa_wait()
1783 	*cs++ = MI_LOAD_REGISTER_REG | (3 - 2);  in alloc_noa_wait()
1792 	*cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4;  in alloc_noa_wait()
1807 	GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch));  in alloc_noa_wait()
1812 	stream->noa_wait = vma;  in alloc_noa_wait()
1818 	if (ret == -EDEADLK) {  in alloc_noa_wait()
1838 					  n_regs - i,  in write_cs_mi_lri()
1875 		return ERR_PTR(-ENOMEM);  in alloc_oa_config_buffer()
1877 	config_length += num_lri_dwords(oa_config->mux_regs_len);  in alloc_oa_config_buffer()
1878 	config_length += num_lri_dwords(oa_config->b_counter_regs_len);  in alloc_oa_config_buffer()
1879 	config_length += num_lri_dwords(oa_config->flex_regs_len);  in alloc_oa_config_buffer()
1883 	obj = i915_gem_object_create_shmem(stream->perf->i915, config_length);  in alloc_oa_config_buffer()
1902 			     oa_config->mux_regs,  in alloc_oa_config_buffer()
1903 			     oa_config->mux_regs_len);  in alloc_oa_config_buffer()
1905 			     oa_config->b_counter_regs,  in alloc_oa_config_buffer()
1906 			     oa_config->b_counter_regs_len);  in alloc_oa_config_buffer()
1908 			     oa_config->flex_regs,  in alloc_oa_config_buffer()
1909 			     oa_config->flex_regs_len);  in alloc_oa_config_buffer()
1912 	*cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ?  in alloc_oa_config_buffer()
1915 	*cs++ = i915_ggtt_offset(stream->noa_wait);  in alloc_oa_config_buffer()
1921 	oa_bo->vma = i915_vma_instance(obj,  in alloc_oa_config_buffer()
1922 				       &stream->engine->gt->ggtt->vm,  in alloc_oa_config_buffer()
1924 	if (IS_ERR(oa_bo->vma)) {  in alloc_oa_config_buffer()
1925 		err = PTR_ERR(oa_bo->vma);  in alloc_oa_config_buffer()
1929 	oa_bo->oa_config = i915_oa_config_get(oa_config);  in alloc_oa_config_buffer()
1930 	llist_add(&oa_bo->node, &stream->oa_config_bos);  in alloc_oa_config_buffer()
1933 	if (err == -EDEADLK) {  in alloc_oa_config_buffer()
1959 	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {  in get_oa_vma()
1960 		if (oa_bo->oa_config == oa_config &&  in get_oa_vma()
1961 		    memcmp(oa_bo->oa_config->uuid,  in get_oa_vma()
1962 			   oa_config->uuid,  in get_oa_vma()
1963 			   sizeof(oa_config->uuid)) == 0)  in get_oa_vma()
1972 	return i915_vma_get(oa_bo->vma);  in get_oa_vma()
1992 	err = i915_gem_object_lock(vma->obj, &ww);  in emit_oa_config()
2000 	intel_engine_pm_get(ce->engine);  in emit_oa_config()
2002 	intel_engine_pm_put(ce->engine);  in emit_oa_config()
2020 	err = i915_request_await_object(rq, vma->obj, 0);  in emit_oa_config()
2026 	err = rq->engine->emit_bb_start(rq,  in emit_oa_config()
2027 					vma->node.start, 0,  in emit_oa_config()
2037 	if (err == -EDEADLK) {  in emit_oa_config()
2050 	return stream->pinned_ctx ?: stream->engine->kernel_context;  in oa_context()
2057 	struct intel_uncore *uncore = stream->uncore;  in hsw_enable_metric_set()
2064 	 * unable to count the events from non-render clock domain.  in hsw_enable_metric_set()
2066 	 * count the events from non-render domain. Unit level clock  in hsw_enable_metric_set()
2075 			      stream->oa_config, oa_context(stream),  in hsw_enable_metric_set()
2081 	struct intel_uncore *uncore = stream->uncore;  in hsw_disable_metric_set()
2105 	for (i = 0; i < oa_config->flex_regs_len; i++) {  in oa_config_flex_reg()
2106 		if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio)  in oa_config_flex_reg()
2107 			return oa_config->flex_regs[i].value;  in oa_config_flex_reg()
2116  * It's fine to put out-of-date values into these per-context registers
2123 	u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset;  in gen8_update_reg_state_unlocked()
2124 	u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;  in gen8_update_reg_state_unlocked()
2135 	u32 *reg_state = ce->lrc_reg_state;  in gen8_update_reg_state_unlocked()
2139 		(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |  in gen8_update_reg_state_unlocked()
2140 		(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |  in gen8_update_reg_state_unlocked()
2145 			oa_config_flex_reg(stream->oa_config, flex_regs[i]);  in gen8_update_reg_state_unlocked()
2166 	offset = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET;  in gen8_store_flex()
2169 		*cs++ = offset + flex->offset * sizeof(u32);  in gen8_store_flex()
2171 		*cs++ = flex->value;  in gen8_store_flex()
2172 	} while (flex++, --count);  in gen8_store_flex()
2194 		*cs++ = i915_mmio_reg_offset(flex->reg);  in gen8_load_flex()
2195 		*cs++ = flex->value;  in gen8_load_flex()
2196 	} while (flex++, --count);  in gen8_load_flex()
2210 	rq = intel_engine_create_kernel_request(ce->engine);  in gen8_modify_context()
2231 	intel_engine_pm_get(ce->engine);  in gen8_modify_self()
2233 	intel_engine_pm_put(ce->engine);  in gen8_modify_self()
2260 		GEM_BUG_ON(ce == ce->engine->kernel_context);  in gen8_configure_context()
2262 		if (ce->engine->class != RENDER_CLASS)  in gen8_configure_context()
2269 		flex->value = intel_sseu_make_rpcs(ce->engine->gt, &ce->sseu);  in gen8_configure_context()
2285 	struct intel_context *ce = stream->pinned_ctx;  in gen12_configure_oar_context()
2286 	u32 format = stream->oa_buffer.format;  in gen12_configure_oar_context()
2290 			stream->perf->ctx_oactxctrl_offset + 1,  in gen12_configure_oar_context()
2306 			RING_CONTEXT_CONTROL(ce->engine->mmio_base),  in gen12_configure_oar_context()
2330  * Manages updating the per-context aspects of the OA stream
2340  * won't automatically reload an out-of-date timer exponent even
2344  * - Ensure the currently running context's per-context OA state is
2346  * - Ensure that all existing contexts will have the correct per-context
2348  * - Ensure any new contexts will be initialized with the correct
2349  *   per-context OA state.
2360 	struct drm_i915_private *i915 = stream->perf->i915;  in oa_configure_all_contexts()
2365 	lockdep_assert_held(&stream->perf->lock);  in oa_configure_all_contexts()
2370 	 * lite-restore). This means we can't safely update a context's image,  in oa_configure_all_contexts()
2383 	spin_lock(&i915->gem.contexts.lock);  in oa_configure_all_contexts()
2384 	list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) {  in oa_configure_all_contexts()
2385 		if (!kref_get_unless_zero(&ctx->ref))  in oa_configure_all_contexts()
2388 		spin_unlock(&i915->gem.contexts.lock);  in oa_configure_all_contexts()
2396 		spin_lock(&i915->gem.contexts.lock);  in oa_configure_all_contexts()
2400 	spin_unlock(&i915->gem.contexts.lock);  in oa_configure_all_contexts()
2408 		struct intel_context *ce = engine->kernel_context;  in oa_configure_all_contexts()
2410 		if (engine->class != RENDER_CLASS)  in oa_configure_all_contexts()
2413 		regs[0].value = intel_sseu_make_rpcs(engine->gt, &ce->sseu);  in oa_configure_all_contexts()
2446 	const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset;  in lrc_configure_all_contexts()
2455 			stream->perf->ctx_oactxctrl_offset + 1,  in lrc_configure_all_contexts()
2469 		(stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) |  in lrc_configure_all_contexts()
2470 		(stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) |  in lrc_configure_all_contexts()
2485 	struct intel_uncore *uncore = stream->uncore;  in gen8_enable_metric_set()
2486 	struct i915_oa_config *oa_config = stream->oa_config;  in gen8_enable_metric_set()
2501 	 * Currently none of the high-level metrics we have depend on knowing  in gen8_enable_metric_set()
2512 	if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) {  in gen8_enable_metric_set()
2528 			      stream->oa_config, oa_context(stream),  in gen8_enable_metric_set()
2535 			     (stream->sample_flags & SAMPLE_OA_REPORT) ?  in oag_report_ctx_switches()
2543 	struct intel_uncore *uncore = stream->uncore;  in gen12_enable_metric_set()
2544 	struct i915_oa_config *oa_config = stream->oa_config;  in gen12_enable_metric_set()
2545 	bool periodic = stream->periodic;  in gen12_enable_metric_set()
2546 	u32 period_exponent = stream->period_exponent;  in gen12_enable_metric_set()
2579 	if (stream->ctx) {  in gen12_enable_metric_set()
2586 			      stream->oa_config, oa_context(stream),  in gen12_enable_metric_set()
2592 	struct intel_uncore *uncore = stream->uncore;  in gen8_disable_metric_set()
2602 	struct intel_uncore *uncore = stream->uncore;  in gen11_disable_metric_set()
2613 	struct intel_uncore *uncore = stream->uncore;  in gen12_disable_metric_set()
2619 	if (stream->ctx)  in gen12_disable_metric_set()
2628 	struct intel_uncore *uncore = stream->uncore;  in gen7_oa_enable()
2629 	struct i915_gem_context *ctx = stream->ctx;  in gen7_oa_enable()
2630 	u32 ctx_id = stream->specific_ctx_id;  in gen7_oa_enable()
2631 	bool periodic = stream->periodic;  in gen7_oa_enable()
2632 	u32 period_exponent = stream->period_exponent;  in gen7_oa_enable()
2633 	u32 report_format = stream->oa_buffer.format;  in gen7_oa_enable()
2658 	struct intel_uncore *uncore = stream->uncore;  in gen8_oa_enable()
2659 	u32 report_format = stream->oa_buffer.format;  in gen8_oa_enable()
2674 	 * filtering and instead filter on the cpu based on the context-id  in gen8_oa_enable()
2684 	struct intel_uncore *uncore = stream->uncore;  in gen12_oa_enable()
2685 	u32 report_format = stream->oa_buffer.format;  in gen12_oa_enable()
2691 	if (!(stream->sample_flags & SAMPLE_OA_REPORT))  in gen12_oa_enable()
2702  * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
2712 	stream->pollin = false;  in i915_oa_stream_enable()
2714 	stream->perf->ops.oa_enable(stream);  in i915_oa_stream_enable()
2716 	if (stream->sample_flags & SAMPLE_OA_REPORT)  in i915_oa_stream_enable()
2717 		hrtimer_start(&stream->poll_check_timer,  in i915_oa_stream_enable()
2718 			      ns_to_ktime(stream->poll_oa_period),  in i915_oa_stream_enable()
2724 	struct intel_uncore *uncore = stream->uncore;  in gen7_oa_disable()
2730 		drm_err(&stream->perf->i915->drm,  in gen7_oa_disable()
2736 	struct intel_uncore *uncore = stream->uncore;  in gen8_oa_disable()
2742 		drm_err(&stream->perf->i915->drm,  in gen8_oa_disable()
2748 	struct intel_uncore *uncore = stream->uncore;  in gen12_oa_disable()
2755 		drm_err(&stream->perf->i915->drm,  in gen12_oa_disable()
2763 		drm_err(&stream->perf->i915->drm,  in gen12_oa_disable()
2768  * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
2777 	stream->perf->ops.oa_disable(stream);  in i915_oa_stream_disable()
2779 	if (stream->sample_flags & SAMPLE_OA_REPORT)  in i915_oa_stream_disable()
2780 		hrtimer_cancel(&stream->poll_check_timer);  in i915_oa_stream_disable()
2799 		return -ENOMEM;  in i915_perf_stream_enable_sync()
2801 	err = stream->perf->ops.enable_metric_set(stream, active);  in i915_perf_stream_enable_sync()
2813 	const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu;  in get_default_sseu_config()
2817 	if (GRAPHICS_VER(engine->i915) == 11) {  in get_default_sseu_config()
2820 		 * we select - just turn off low bits in the amount of half of  in get_default_sseu_config()
2823 		out_sseu->subslice_mask =  in get_default_sseu_config()
2824 			~(~0 << (hweight8(out_sseu->subslice_mask) / 2));  in get_default_sseu_config()
2825 		out_sseu->slice_mask = 0x1;  in get_default_sseu_config()
2834 	if (drm_sseu->engine.engine_class != engine->uabi_class ||  in get_sseu_config()
2835 	    drm_sseu->engine.engine_instance != engine->uabi_instance)  in get_sseu_config()
2836 		return -EINVAL;  in get_sseu_config()
2838 	return i915_gem_user_to_context_sseu(engine->gt, drm_sseu, out_sseu);  in get_sseu_config()
2842  * i915_oa_stream_init - validate combined props for OA stream and init
2863 	struct drm_i915_private *i915 = stream->perf->i915;  in i915_oa_stream_init()
2864 	struct i915_perf *perf = stream->perf;  in i915_oa_stream_init()
2868 	if (!props->engine) {  in i915_oa_stream_init()
2869 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2871 		return -EINVAL;  in i915_oa_stream_init()
2879 	if (!perf->metrics_kobj) {  in i915_oa_stream_init()
2880 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2882 		return -EINVAL;  in i915_oa_stream_init()
2885 	if (!(props->sample_flags & SAMPLE_OA_REPORT) &&  in i915_oa_stream_init()
2886 	    (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) {  in i915_oa_stream_init()
2887 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2889 		return -EINVAL;  in i915_oa_stream_init()
2892 	if (!perf->ops.enable_metric_set) {  in i915_oa_stream_init()
2893 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2895 		return -ENODEV;  in i915_oa_stream_init()
2903 	if (perf->exclusive_stream) {  in i915_oa_stream_init()
2904 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2906 		return -EBUSY;  in i915_oa_stream_init()
2909 	if (!props->oa_format) {  in i915_oa_stream_init()
2910 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2912 		return -EINVAL;  in i915_oa_stream_init()
2915 	stream->engine = props->engine;  in i915_oa_stream_init()
2916 	stream->uncore = stream->engine->gt->uncore;  in i915_oa_stream_init()
2918 	stream->sample_size = sizeof(struct drm_i915_perf_record_header);  in i915_oa_stream_init()
2920 	format_size = perf->oa_formats[props->oa_format].size;  in i915_oa_stream_init()
2922 	stream->sample_flags = props->sample_flags;  in i915_oa_stream_init()
2923 	stream->sample_size += format_size;  in i915_oa_stream_init()
2925 	stream->oa_buffer.format_size = format_size;  in i915_oa_stream_init()
2926 	if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0))  in i915_oa_stream_init()
2927 		return -EINVAL;  in i915_oa_stream_init()
2929 	stream->hold_preemption = props->hold_preemption;  in i915_oa_stream_init()
2931 	stream->oa_buffer.format =  in i915_oa_stream_init()
2932 		perf->oa_formats[props->oa_format].format;  in i915_oa_stream_init()
2934 	stream->periodic = props->oa_periodic;  in i915_oa_stream_init()
2935 	if (stream->periodic)  in i915_oa_stream_init()
2936 		stream->period_exponent = props->oa_period_exponent;  in i915_oa_stream_init()
2938 	if (stream->ctx) {  in i915_oa_stream_init()
2941 			drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2949 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2954 	stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set);  in i915_oa_stream_init()
2955 	if (!stream->oa_config) {  in i915_oa_stream_init()
2956 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2957 			"Invalid OA config id=%i\n", props->metrics_set);  in i915_oa_stream_init()
2958 		ret = -EINVAL;  in i915_oa_stream_init()
2962 	/* PRM - observability performance counters:  in i915_oa_stream_init()
2974 	intel_engine_pm_get(stream->engine);  in i915_oa_stream_init()
2975 	intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL);  in i915_oa_stream_init()
2981 	stream->ops = &i915_oa_stream_ops;  in i915_oa_stream_init()
2983 	perf->sseu = props->sseu;  in i915_oa_stream_init()
2984 	WRITE_ONCE(perf->exclusive_stream, stream);  in i915_oa_stream_init()
2988 		drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2993 	drm_dbg(&stream->perf->i915->drm,  in i915_oa_stream_init()
2995 		  stream->oa_config->uuid);  in i915_oa_stream_init()
2997 	hrtimer_init(&stream->poll_check_timer,  in i915_oa_stream_init()
2999 	stream->poll_check_timer.function = oa_poll_check_timer_cb;  in i915_oa_stream_init()
3000 	init_waitqueue_head(&stream->poll_wq);  in i915_oa_stream_init()
3001 	spin_lock_init(&stream->oa_buffer.ptr_lock);  in i915_oa_stream_init()
3006 	WRITE_ONCE(perf->exclusive_stream, NULL);  in i915_oa_stream_init()
3007 	perf->ops.disable_metric_set(stream);  in i915_oa_stream_init()
3014 	intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL);  in i915_oa_stream_init()
3015 	intel_engine_pm_put(stream->engine);  in i915_oa_stream_init()
3021 	if (stream->ctx)  in i915_oa_stream_init()
3032 	if (engine->class != RENDER_CLASS)  in i915_oa_init_reg_state()
3036 	stream = READ_ONCE(engine->i915->perf.exclusive_stream);  in i915_oa_init_reg_state()
3037 	if (stream && GRAPHICS_VER(stream->perf->i915) < 12)  in i915_oa_init_reg_state()
3042  * i915_perf_read - handles read() FOP for i915 perf stream FDs
3050  * &i915_perf_stream_ops->read but to save having stream implementations (of
3064 	struct i915_perf_stream *stream = file->private_data;  in i915_perf_read()
3065 	struct i915_perf *perf = stream->perf;  in i915_perf_read()
3073 	if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT))  in i915_perf_read()
3074 		return -EIO;  in i915_perf_read()
3076 	if (!(file->f_flags & O_NONBLOCK)) {  in i915_perf_read()
3078 		 * stream->ops->wait_unlocked.  in i915_perf_read()
3085 			ret = stream->ops->wait_unlocked(stream);  in i915_perf_read()
3089 			mutex_lock(&perf->lock);  in i915_perf_read()
3090 			ret = stream->ops->read(stream, buf, count, &offset);  in i915_perf_read()
3091 			mutex_unlock(&perf->lock);  in i915_perf_read()
3094 		mutex_lock(&perf->lock);  in i915_perf_read()
3095 		ret = stream->ops->read(stream, buf, count, &offset);  in i915_perf_read()
3096 		mutex_unlock(&perf->lock);  in i915_perf_read()
3103 	 * and read() returning -EAGAIN. Clearing the oa.pollin state here  in i915_perf_read()
3106 	 * The exception to this is if ops->read() returned -ENOSPC which means  in i915_perf_read()
3110 	if (ret != -ENOSPC)  in i915_perf_read()
3111 		stream->pollin = false;  in i915_perf_read()
3113 	/* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */  in i915_perf_read()
3114 	return offset ?: (ret ?: -EAGAIN);  in i915_perf_read()
3123 		stream->pollin = true;  in oa_poll_check_timer_cb()
3124 		wake_up(&stream->poll_wq);  in oa_poll_check_timer_cb()
3128 			    ns_to_ktime(stream->poll_oa_period));  in oa_poll_check_timer_cb()
3134  * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3140  * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
3143  * Note: The &perf->lock mutex has been taken to serialize
3144  * with any non-file-operation driver hooks.
3154 	stream->ops->poll_wait(stream, file, wait);  in i915_perf_poll_locked()
3162 	if (stream->pollin)  in i915_perf_poll_locked()
3169  * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3183 	struct i915_perf_stream *stream = file->private_data;  in i915_perf_poll()
3184 	struct i915_perf *perf = stream->perf;  in i915_perf_poll()
3187 	mutex_lock(&perf->lock);  in i915_perf_poll()
3189 	mutex_unlock(&perf->lock);  in i915_perf_poll()
3195  * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
3206 	if (stream->enabled)  in i915_perf_enable_locked()
3209 	/* Allow stream->ops->enable() to refer to this */  in i915_perf_enable_locked()
3210 	stream->enabled = true;  in i915_perf_enable_locked()
3212 	if (stream->ops->enable)  in i915_perf_enable_locked()
3213 		stream->ops->enable(stream);  in i915_perf_enable_locked()
3215 	if (stream->hold_preemption)  in i915_perf_enable_locked()
3216 		intel_context_set_nopreempt(stream->pinned_ctx);  in i915_perf_enable_locked()
3220  * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
3225  * The intention is that disabling an re-enabling a stream will ideally be
3226  * cheaper than destroying and re-opening a stream with the same configuration,
3228  * must be retained between disabling and re-enabling a stream.
3231  * to attempt to read from the stream (-EIO).
3235 	if (!stream->enabled)  in i915_perf_disable_locked()
3238 	/* Allow stream->ops->disable() to refer to this */  in i915_perf_disable_locked()
3239 	stream->enabled = false;  in i915_perf_disable_locked()
3241 	if (stream->hold_preemption)  in i915_perf_disable_locked()
3242 		intel_context_clear_nopreempt(stream->pinned_ctx);  in i915_perf_disable_locked()
3244 	if (stream->ops->disable)  in i915_perf_disable_locked()
3245 		stream->ops->disable(stream);  in i915_perf_disable_locked()
3252 	long ret = stream->oa_config->id;  in i915_perf_config_locked()
3254 	config = i915_perf_get_oa_config(stream->perf, metrics_set);  in i915_perf_config_locked()
3256 		return -EINVAL;  in i915_perf_config_locked()
3258 	if (config != stream->oa_config) {  in i915_perf_config_locked()
3272 			config = xchg(&stream->oa_config, config);  in i915_perf_config_locked()
3283  * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3288  * Note: The &perf->lock mutex has been taken to serialize
3289  * with any non-file-operation driver hooks.
3291  * Returns: zero on success or a negative error code. Returns -EINVAL for
3309 	return -EINVAL;  in i915_perf_ioctl_locked()
3313  * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3320  * Returns: zero on success or a negative error code. Returns -EINVAL for
3327 	struct i915_perf_stream *stream = file->private_data;  in i915_perf_ioctl()
3328 	struct i915_perf *perf = stream->perf;  in i915_perf_ioctl()
3331 	mutex_lock(&perf->lock);  in i915_perf_ioctl()
3333 	mutex_unlock(&perf->lock);  in i915_perf_ioctl()
3339  * i915_perf_destroy_locked - destroy an i915 perf stream
3345  * Note: The &perf->lock mutex has been taken to serialize
3346  * with any non-file-operation driver hooks.
3350 	if (stream->enabled)  in i915_perf_destroy_locked()
3353 	if (stream->ops->destroy)  in i915_perf_destroy_locked()
3354 		stream->ops->destroy(stream);  in i915_perf_destroy_locked()
3356 	if (stream->ctx)  in i915_perf_destroy_locked()
3357 		i915_gem_context_put(stream->ctx);  in i915_perf_destroy_locked()
3363  * i915_perf_release - handles userspace close() of a stream file
3375 	struct i915_perf_stream *stream = file->private_data;  in i915_perf_release()
3376 	struct i915_perf *perf = stream->perf;  in i915_perf_release()
3378 	mutex_lock(&perf->lock);  in i915_perf_release()
3380 	mutex_unlock(&perf->lock);  in i915_perf_release()
3383 	drm_dev_put(&perf->i915->drm);  in i915_perf_release()
3404  * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3413  * behalf of i915_perf_open_ioctl() with the &perf->lock mutex
3414  * taken to serialize with any non-file-operation driver hooks.
3440 	if (props->single_context) {  in i915_perf_open_ioctl_locked()
3441 		u32 ctx_handle = props->ctx_handle;  in i915_perf_open_ioctl_locked()
3442 		struct drm_i915_file_private *file_priv = file->driver_priv;  in i915_perf_open_ioctl_locked()
3446 			drm_dbg(&perf->i915->drm,  in i915_perf_open_ioctl_locked()
3458 	 * non-privileged client.  in i915_perf_open_ioctl_locked()
3460 	 * For Gen8->11 the OA unit no longer supports clock gating off for a  in i915_perf_open_ioctl_locked()
3462 	 * from updating as system-wide / global values. Even though we can  in i915_perf_open_ioctl_locked()
3473 	if (IS_HASWELL(perf->i915) && specific_ctx)  in i915_perf_open_ioctl_locked()
3475 	else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx &&  in i915_perf_open_ioctl_locked()
3476 		 (props->sample_flags & SAMPLE_OA_REPORT) == 0)  in i915_perf_open_ioctl_locked()
3479 	if (props->hold_preemption) {  in i915_perf_open_ioctl_locked()
3480 		if (!props->single_context) {  in i915_perf_open_ioctl_locked()
3481 			drm_dbg(&perf->i915->drm,  in i915_perf_open_ioctl_locked()
3483 			ret = -EINVAL;  in i915_perf_open_ioctl_locked()
3492 	if (props->has_sseu)  in i915_perf_open_ioctl_locked()
3495 		get_default_sseu_config(&props->sseu, props->engine);  in i915_perf_open_ioctl_locked()
3504 		drm_dbg(&perf->i915->drm,  in i915_perf_open_ioctl_locked()
3506 		ret = -EACCES;  in i915_perf_open_ioctl_locked()
3512 		ret = -ENOMEM;  in i915_perf_open_ioctl_locked()
3516 	stream->perf = perf;  in i915_perf_open_ioctl_locked()
3517 	stream->ctx = specific_ctx;  in i915_perf_open_ioctl_locked()
3518 	stream->poll_oa_period = props->poll_oa_period;  in i915_perf_open_ioctl_locked()
3524 	/* we avoid simply assigning stream->sample_flags = props->sample_flags  in i915_perf_open_ioctl_locked()
3528 	if (WARN_ON(stream->sample_flags != props->sample_flags)) {  in i915_perf_open_ioctl_locked()
3529 		ret = -ENODEV;  in i915_perf_open_ioctl_locked()
3533 	if (param->flags & I915_PERF_FLAG_FD_CLOEXEC)  in i915_perf_open_ioctl_locked()
3535 	if (param->flags & I915_PERF_FLAG_FD_NONBLOCK)  in i915_perf_open_ioctl_locked()
3544 	if (!(param->flags & I915_PERF_FLAG_DISABLED))  in i915_perf_open_ioctl_locked()
3550 	drm_dev_get(&perf->i915->drm);  in i915_perf_open_ioctl_locked()
3555 	if (stream->ops->destroy)  in i915_perf_open_ioctl_locked()
3556 		stream->ops->destroy(stream);  in i915_perf_open_ioctl_locked()
3568 	return intel_gt_clock_interval_to_ns(to_gt(perf->i915),  in oa_exponent_to_ns()
3575 	return test_bit(format, perf->format_mask);  in oa_format_valid()
3581 	__set_bit(format, perf->format_mask);  in oa_format_add()
3585  * read_properties_unlocked - validate + copy userspace stream open properties
3609 	props->poll_oa_period = DEFAULT_POLL_PERIOD_NS;  in read_properties_unlocked()
3612 		drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3614 		return -EINVAL;  in read_properties_unlocked()
3617 	/* At the moment we only support using i915-perf on the RCS. */  in read_properties_unlocked()
3618 	props->engine = intel_engine_lookup_user(perf->i915,  in read_properties_unlocked()
3621 	if (!props->engine) {  in read_properties_unlocked()
3622 		drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3623 			"No RENDER-capable engines\n");  in read_properties_unlocked()
3624 		return -EINVAL;  in read_properties_unlocked()
3634 		drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3636 		return -EINVAL;  in read_properties_unlocked()
3652 			drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3654 			return -EINVAL;  in read_properties_unlocked()
3659 			props->single_context = 1;  in read_properties_unlocked()
3660 			props->ctx_handle = value;  in read_properties_unlocked()
3664 				props->sample_flags |= SAMPLE_OA_REPORT;  in read_properties_unlocked()
3668 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3670 				return -EINVAL;  in read_properties_unlocked()
3672 			props->metrics_set = value;  in read_properties_unlocked()
3676 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3677 					"Out-of-range OA report format %llu\n",  in read_properties_unlocked()
3679 				return -EINVAL;  in read_properties_unlocked()
3682 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3685 				return -EINVAL;  in read_properties_unlocked()
3687 			props->oa_format = value;  in read_properties_unlocked()
3691 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3694 				return -EINVAL;  in read_properties_unlocked()
3720 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3723 				return -EACCES;  in read_properties_unlocked()
3726 			props->oa_periodic = true;  in read_properties_unlocked()
3727 			props->oa_period_exponent = value;  in read_properties_unlocked()
3730 			props->hold_preemption = !!value;  in read_properties_unlocked()
3735 			if (GRAPHICS_VER_FULL(perf->i915) >= IP_VER(12, 50)) {  in read_properties_unlocked()
3736 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3738 					GRAPHICS_VER_FULL(perf->i915));  in read_properties_unlocked()
3739 				return -ENODEV;  in read_properties_unlocked()
3745 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3747 				return -EFAULT;  in read_properties_unlocked()
3750 			ret = get_sseu_config(&props->sseu, props->engine, &user_sseu);  in read_properties_unlocked()
3752 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3756 			props->has_sseu = true;  in read_properties_unlocked()
3761 				drm_dbg(&perf->i915->drm,  in read_properties_unlocked()
3764 				return -EINVAL;  in read_properties_unlocked()
3766 			props->poll_oa_period = value;  in read_properties_unlocked()
3770 			return -EINVAL;  in read_properties_unlocked()
3780  * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
3790  * i915-perf stream is expected to be a suitable interface for other forms of
3797  * i915_perf_open_ioctl_locked() after taking the &perf->lock
3798  * mutex for serializing with any non-file-operation driver hooks.
3806 	struct i915_perf *perf = &to_i915(dev)->perf;  in i915_perf_open_ioctl()
3812 	if (!perf->i915) {  in i915_perf_open_ioctl()
3813 		drm_dbg(&perf->i915->drm,  in i915_perf_open_ioctl()
3815 		return -ENOTSUPP;  in i915_perf_open_ioctl()
3821 	if (param->flags & ~known_open_flags) {  in i915_perf_open_ioctl()
3822 		drm_dbg(&perf->i915->drm,  in i915_perf_open_ioctl()
3824 		return -EINVAL;  in i915_perf_open_ioctl()
3828 				       u64_to_user_ptr(param->properties_ptr),  in i915_perf_open_ioctl()
3829 				       param->num_properties,  in i915_perf_open_ioctl()
3834 	mutex_lock(&perf->lock);  in i915_perf_open_ioctl()
3836 	mutex_unlock(&perf->lock);  in i915_perf_open_ioctl()
3842  * i915_perf_register - exposes i915-perf to userspace
3847  * used to open an i915-perf stream.
3851 	struct i915_perf *perf = &i915->perf;  in i915_perf_register()
3853 	if (!perf->i915)  in i915_perf_register()
3860 	mutex_lock(&perf->lock);  in i915_perf_register()
3862 	perf->metrics_kobj =  in i915_perf_register()
3864 				       &i915->drm.primary->kdev->kobj);  in i915_perf_register()
3866 	mutex_unlock(&perf->lock);  in i915_perf_register()
3870  * i915_perf_unregister - hide i915-perf from userspace
3873  * i915-perf state cleanup is split up into an 'unregister' and
3880 	struct i915_perf *perf = &i915->perf;  in i915_perf_unregister()
3882 	if (!perf->metrics_kobj)  in i915_perf_unregister()
3885 	kobject_put(perf->metrics_kobj);  in i915_perf_unregister()
3886 	perf->metrics_kobj = NULL;  in i915_perf_unregister()
3911 	while (table->start || table->end) {  in reg_in_range_table()
3912 		if (addr >= table->start && addr <= table->end)  in reg_in_range_table()
3925 	{ .start = 0x2710, .end = 0x272c },	/* OASTARTTRIG[1-8] */
3926 	{ .start = 0x2740, .end = 0x275c },	/* OAREPORTTRIG[1-8] */
3927 	{ .start = 0x2770, .end = 0x27ac },	/* OACEC[0-7][0-1] */
3933 	{ .start = 0xd900, .end = 0xd91c },	/* GEN12_OAG_OASTARTTRIG[1-8] */
3934 	{ .start = 0xd920, .end = 0xd93c },	/* GEN12_OAG_OAREPORTTRIG1[1-8] */
3935 	{ .start = 0xd940, .end = 0xd97c },	/* GEN12_OAG_CEC[0-7][0-1] */
3936 	{ .start = 0xdc00, .end = 0xdc3c },	/* GEN12_OAG_SCEC[0-7][0-1] */
3943 	{ .start = 0x91b8, .end = 0x91cc },	/* OA_PERFCNT[1-2], OA_PERFMATRIX */
3944 	{ .start = 0x9800, .end = 0x9888 },	/* MICRO_BP0_0 - NOA_WRITE */
3950 	{ .start = 0x09e80, .end = 0x09ea4 }, /* HSW_MBVID2_NOA[0-9] */
3962 	{ .start = 0x0d00, .end = 0x0d2c },	/* RPM_CONFIG[0-1], NOA_CONFIG[0-8] */
3968 	{ .start = 0x91c8, .end = 0x91dc },	/* OA_PERFCNT[3-4] */
3973 	{ .start = 0x0d00, .end = 0x0d04 },     /* RPM_CONFIG[0-1] */
3974 	{ .start = 0x0d0c, .end = 0x0d2c },     /* NOA_CONFIG[0-8] */
4055 		return ERR_PTR(-EINVAL);  in alloc_oa_regs()
4059 		return ERR_PTR(-ENOMEM);  in alloc_oa_regs()
4069 			drm_dbg(&perf->i915->drm,  in alloc_oa_regs()
4071 			err = -EINVAL;  in alloc_oa_regs()
4099 	return sprintf(buf, "%d\n", oa_config->id);  in show_dynamic_id()
4105 	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);  in create_dynamic_oa_sysfs_entry()
4106 	oa_config->sysfs_metric_id.attr.name = "id";  in create_dynamic_oa_sysfs_entry()
4107 	oa_config->sysfs_metric_id.attr.mode = S_IRUGO;  in create_dynamic_oa_sysfs_entry()
4108 	oa_config->sysfs_metric_id.show = show_dynamic_id;  in create_dynamic_oa_sysfs_entry()
4109 	oa_config->sysfs_metric_id.store = NULL;  in create_dynamic_oa_sysfs_entry()
4111 	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;  in create_dynamic_oa_sysfs_entry()
4112 	oa_config->attrs[1] = NULL;  in create_dynamic_oa_sysfs_entry()
4114 	oa_config->sysfs_metric.name = oa_config->uuid;  in create_dynamic_oa_sysfs_entry()
4115 	oa_config->sysfs_metric.attrs = oa_config->attrs;  in create_dynamic_oa_sysfs_entry()
4117 	return sysfs_create_group(perf->metrics_kobj,  in create_dynamic_oa_sysfs_entry()
4118 				  &oa_config->sysfs_metric);  in create_dynamic_oa_sysfs_entry()
4122  * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
4137 	struct i915_perf *perf = &to_i915(dev)->perf;  in i915_perf_add_config_ioctl()
4143 	if (!perf->i915) {  in i915_perf_add_config_ioctl()
4144 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4146 		return -ENOTSUPP;  in i915_perf_add_config_ioctl()
4149 	if (!perf->metrics_kobj) {  in i915_perf_add_config_ioctl()
4150 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4152 		return -EINVAL;  in i915_perf_add_config_ioctl()
4156 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4158 		return -EACCES;  in i915_perf_add_config_ioctl()
4161 	if ((!args->mux_regs_ptr || !args->n_mux_regs) &&  in i915_perf_add_config_ioctl()
4162 	    (!args->boolean_regs_ptr || !args->n_boolean_regs) &&  in i915_perf_add_config_ioctl()
4163 	    (!args->flex_regs_ptr || !args->n_flex_regs)) {  in i915_perf_add_config_ioctl()
4164 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4166 		return -EINVAL;  in i915_perf_add_config_ioctl()
4171 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4173 		return -ENOMEM;  in i915_perf_add_config_ioctl()
4176 	oa_config->perf = perf;  in i915_perf_add_config_ioctl()
4177 	kref_init(&oa_config->ref);  in i915_perf_add_config_ioctl()
4179 	if (!uuid_is_valid(args->uuid)) {  in i915_perf_add_config_ioctl()
4180 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4182 		err = -EINVAL;  in i915_perf_add_config_ioctl()
4186 	/* Last character in oa_config->uuid will be 0 because oa_config is  in i915_perf_add_config_ioctl()
4189 	memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid));  in i915_perf_add_config_ioctl()
4191 	oa_config->mux_regs_len = args->n_mux_regs;  in i915_perf_add_config_ioctl()
4193 			     perf->ops.is_valid_mux_reg,  in i915_perf_add_config_ioctl()
4194 			     u64_to_user_ptr(args->mux_regs_ptr),  in i915_perf_add_config_ioctl()
4195 			     args->n_mux_regs);  in i915_perf_add_config_ioctl()
4198 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4203 	oa_config->mux_regs = regs;  in i915_perf_add_config_ioctl()
4205 	oa_config->b_counter_regs_len = args->n_boolean_regs;  in i915_perf_add_config_ioctl()
4207 			     perf->ops.is_valid_b_counter_reg,  in i915_perf_add_config_ioctl()
4208 			     u64_to_user_ptr(args->boolean_regs_ptr),  in i915_perf_add_config_ioctl()
4209 			     args->n_boolean_regs);  in i915_perf_add_config_ioctl()
4212 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4217 	oa_config->b_counter_regs = regs;  in i915_perf_add_config_ioctl()
4219 	if (GRAPHICS_VER(perf->i915) < 8) {  in i915_perf_add_config_ioctl()
4220 		if (args->n_flex_regs != 0) {  in i915_perf_add_config_ioctl()
4221 			err = -EINVAL;  in i915_perf_add_config_ioctl()
4225 		oa_config->flex_regs_len = args->n_flex_regs;  in i915_perf_add_config_ioctl()
4227 				     perf->ops.is_valid_flex_reg,  in i915_perf_add_config_ioctl()
4228 				     u64_to_user_ptr(args->flex_regs_ptr),  in i915_perf_add_config_ioctl()
4229 				     args->n_flex_regs);  in i915_perf_add_config_ioctl()
4232 			drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4237 		oa_config->flex_regs = regs;  in i915_perf_add_config_ioctl()
4240 	err = mutex_lock_interruptible(&perf->metrics_lock);  in i915_perf_add_config_ioctl()
4247 	idr_for_each_entry(&perf->metrics_idr, tmp, id) {  in i915_perf_add_config_ioctl()
4248 		if (!strcmp(tmp->uuid, oa_config->uuid)) {  in i915_perf_add_config_ioctl()
4249 			drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4251 			err = -EADDRINUSE;  in i915_perf_add_config_ioctl()
4258 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4264 	oa_config->id = idr_alloc(&perf->metrics_idr,  in i915_perf_add_config_ioctl()
4267 	if (oa_config->id < 0) {  in i915_perf_add_config_ioctl()
4268 		drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4270 		err = oa_config->id;  in i915_perf_add_config_ioctl()
4274 	mutex_unlock(&perf->metrics_lock);  in i915_perf_add_config_ioctl()
4276 	drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4277 		"Added config %s id=%i\n", oa_config->uuid, oa_config->id);  in i915_perf_add_config_ioctl()
4279 	return oa_config->id;  in i915_perf_add_config_ioctl()
4282 	mutex_unlock(&perf->metrics_lock);  in i915_perf_add_config_ioctl()
4285 	drm_dbg(&perf->i915->drm,  in i915_perf_add_config_ioctl()
4291  * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
4304 	struct i915_perf *perf = &to_i915(dev)->perf;  in i915_perf_remove_config_ioctl()
4309 	if (!perf->i915) {  in i915_perf_remove_config_ioctl()
4310 		drm_dbg(&perf->i915->drm,  in i915_perf_remove_config_ioctl()
4312 		return -ENOTSUPP;  in i915_perf_remove_config_ioctl()
4316 		drm_dbg(&perf->i915->drm,  in i915_perf_remove_config_ioctl()
4318 		return -EACCES;  in i915_perf_remove_config_ioctl()
4321 	ret = mutex_lock_interruptible(&perf->metrics_lock);  in i915_perf_remove_config_ioctl()
4325 	oa_config = idr_find(&perf->metrics_idr, *arg);  in i915_perf_remove_config_ioctl()
4327 		drm_dbg(&perf->i915->drm,  in i915_perf_remove_config_ioctl()
4329 		ret = -ENOENT;  in i915_perf_remove_config_ioctl()
4333 	GEM_BUG_ON(*arg != oa_config->id);  in i915_perf_remove_config_ioctl()
4335 	sysfs_remove_group(perf->metrics_kobj, &oa_config->sysfs_metric);  in i915_perf_remove_config_ioctl()
4337 	idr_remove(&perf->metrics_idr, *arg);  in i915_perf_remove_config_ioctl()
4339 	mutex_unlock(&perf->metrics_lock);  in i915_perf_remove_config_ioctl()
4341 	drm_dbg(&perf->i915->drm,  in i915_perf_remove_config_ioctl()
4342 		"Removed config %s id=%i\n", oa_config->uuid, oa_config->id);  in i915_perf_remove_config_ioctl()
4349 	mutex_unlock(&perf->metrics_lock);  in i915_perf_remove_config_ioctl()
4377 	struct drm_i915_private *i915 = perf->i915;  in oa_init_supported_formats()
4378 	enum intel_platform platform = INTEL_INFO(i915)->platform;  in oa_init_supported_formats()
4420  * i915_perf_init - initialize i915-perf state on module bind
4423  * Initializes i915-perf state without exposing anything to userspace.
4425  * Note: i915-perf initialization is split into an 'init' and 'register'
4430 	struct i915_perf *perf = &i915->perf;  in i915_perf_init()
4438 	perf->oa_formats = oa_formats;  in i915_perf_init()
4440 		perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr;  in i915_perf_init()
4441 		perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr;  in i915_perf_init()
4442 		perf->ops.is_valid_flex_reg = NULL;  in i915_perf_init()
4443 		perf->ops.enable_metric_set = hsw_enable_metric_set;  in i915_perf_init()
4444 		perf->ops.disable_metric_set = hsw_disable_metric_set;  in i915_perf_init()
4445 		perf->ops.oa_enable = gen7_oa_enable;  in i915_perf_init()
4446 		perf->ops.oa_disable = gen7_oa_disable;  in i915_perf_init()
4447 		perf->ops.read = gen7_oa_read;  in i915_perf_init()
4448 		perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read;  in i915_perf_init()
4456 		perf->ops.read = gen8_oa_read;  in i915_perf_init()
4459 			perf->ops.is_valid_b_counter_reg =  in i915_perf_init()
4461 			perf->ops.is_valid_mux_reg =  in i915_perf_init()
4463 			perf->ops.is_valid_flex_reg =  in i915_perf_init()
4467 				perf->ops.is_valid_mux_reg =  in i915_perf_init()
4471 			perf->ops.oa_enable = gen8_oa_enable;  in i915_perf_init()
4472 			perf->ops.oa_disable = gen8_oa_disable;  in i915_perf_init()
4473 			perf->ops.enable_metric_set = gen8_enable_metric_set;  in i915_perf_init()
4474 			perf->ops.disable_metric_set = gen8_disable_metric_set;  in i915_perf_init()
4475 			perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;  in i915_perf_init()
4478 				perf->ctx_oactxctrl_offset = 0x120;  in i915_perf_init()
4479 				perf->ctx_flexeu0_offset = 0x2ce;  in i915_perf_init()
4481 				perf->gen8_valid_ctx_bit = BIT(25);  in i915_perf_init()
4483 				perf->ctx_oactxctrl_offset = 0x128;  in i915_perf_init()
4484 				perf->ctx_flexeu0_offset = 0x3de;  in i915_perf_init()
4486 				perf->gen8_valid_ctx_bit = BIT(16);  in i915_perf_init()
4489 			perf->ops.is_valid_b_counter_reg =  in i915_perf_init()
4491 			perf->ops.is_valid_mux_reg =  in i915_perf_init()
4493 			perf->ops.is_valid_flex_reg =  in i915_perf_init()
4496 			perf->ops.oa_enable = gen8_oa_enable;  in i915_perf_init()
4497 			perf->ops.oa_disable = gen8_oa_disable;  in i915_perf_init()
4498 			perf->ops.enable_metric_set = gen8_enable_metric_set;  in i915_perf_init()
4499 			perf->ops.disable_metric_set = gen11_disable_metric_set;  in i915_perf_init()
4500 			perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read;  in i915_perf_init()
4502 			perf->ctx_oactxctrl_offset = 0x124;  in i915_perf_init()
4503 			perf->ctx_flexeu0_offset = 0x78e;  in i915_perf_init()
4505 			perf->gen8_valid_ctx_bit = BIT(16);  in i915_perf_init()
4507 			perf->ops.is_valid_b_counter_reg =  in i915_perf_init()
4509 			perf->ops.is_valid_mux_reg =  in i915_perf_init()
4511 			perf->ops.is_valid_flex_reg =  in i915_perf_init()
4514 			perf->ops.oa_enable = gen12_oa_enable;  in i915_perf_init()
4515 			perf->ops.oa_disable = gen12_oa_disable;  in i915_perf_init()
4516 			perf->ops.enable_metric_set = gen12_enable_metric_set;  in i915_perf_init()
4517 			perf->ops.disable_metric_set = gen12_disable_metric_set;  in i915_perf_init()
4518 			perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read;  in i915_perf_init()
4520 			perf->ctx_flexeu0_offset = 0;  in i915_perf_init()
4521 			perf->ctx_oactxctrl_offset = 0x144;  in i915_perf_init()
4525 	if (perf->ops.enable_metric_set) {  in i915_perf_init()
4526 		mutex_init(&perf->lock);  in i915_perf_init()
4529 		oa_sample_rate_hard_limit = to_gt(i915)->clock_frequency / 2;  in i915_perf_init()
4531 		mutex_init(&perf->metrics_lock);  in i915_perf_init()
4532 		idr_init_base(&perf->metrics_idr, 1);  in i915_perf_init()
4544 		ratelimit_state_init(&perf->spurious_report_rs, 5 * HZ, 10);  in i915_perf_init()
4549 		ratelimit_set_flags(&perf->spurious_report_rs,  in i915_perf_init()
4552 		ratelimit_state_init(&perf->tail_pointer_race,  in i915_perf_init()
4554 		ratelimit_set_flags(&perf->tail_pointer_race,  in i915_perf_init()
4557 		atomic64_set(&perf->noa_programming_delay,  in i915_perf_init()
4560 		perf->i915 = i915;  in i915_perf_init()
4584  * i915_perf_fini - Counter part to i915_perf_init()
4589 	struct i915_perf *perf = &i915->perf;  in i915_perf_fini()
4591 	if (!perf->i915)  in i915_perf_fini()
4594 	idr_for_each(&perf->metrics_idr, destroy_config, perf);  in i915_perf_fini()
4595 	idr_destroy(&perf->metrics_idr);  in i915_perf_fini()
4597 	memset(&perf->ops, 0, sizeof(perf->ops));  in i915_perf_fini()
4598 	perf->i915 = NULL;  in i915_perf_fini()
4602  * i915_perf_ioctl_version - Version of the i915-perf subsystem