Lines Matching +full:cpu +full:- +full:centric
2 * Copyright © 2015-2016 Intel Corporation
40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
44 * without special privileges. Access to system-wide metrics requires root
58 * might sample sets of tightly-coupled counters, depending on the
70 * interleaved with event-type specific members.
74 * with the CPU, using HW specific packing formats for counter sets. Sometimes
76 * would be acceptable to expose them to unprivileged applications - to hide
88 * into perf's currently cpu centric design.
96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
102 * For posterity, in case we might re-visit trying to adapt core perf to be
106 * - The perf based OA PMU driver broke some significant design assumptions:
108 * Existing perf pmus are used for profiling work on a cpu and we were
110 * implications, the need to fake cpu-related data (such as user/kernel
112 * as a way to forward device-specific status records.
115 * involvement from the CPU, making our PMU driver the first of a kind.
117 * Given the way we were periodically forward data from the GPU-mapped, OA
123 * explicitly initiated from the cpu (say in response to a userspace read())
125 * trigger a report from the cpu on demand.
130 * opened, there's no clear precedent for being able to provide group-wide
139 * for combining with the side-band raw reports it captures using
142 * - As a side note on perf's grouping feature; there was also some concern
158 * one time. The OA unit is not designed to allow re-configuration while in
178 * - It felt like our perf based PMU was making some technical compromises
182 * cpu core, while our device pmu related to neither. Events opened with a
184 * that process - so not appropriate for us. When an event is related to a
185 * cpu id, perf ensures pmu methods will be invoked via an inter process
187 * perf events for a specific cpu. This was workable but it meant the
217 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
225 * CPU).
228 * by checking for a zeroed report-id field in tail reports, we want to account
248 * non-periodic reports (such as on context switch) or the OA unit may be
302 * code assumes all reports have a power-of-two size and ~(size - 1) can
322 * struct perf_open_properties - for validated properties given to open a stream
337 * @poll_oa_period: The period in nanoseconds at which the CPU will check for OA
381 kfree(oa_config->flex_regs); in i915_oa_config_release()
382 kfree(oa_config->b_counter_regs); in i915_oa_config_release()
383 kfree(oa_config->mux_regs); in i915_oa_config_release()
394 oa_config = idr_find(&perf->metrics_idr, metrics_set); in i915_perf_get_oa_config()
404 i915_oa_config_put(oa_bo->oa_config); in free_oa_config_bo()
405 i915_vma_put(oa_bo->vma); in free_oa_config_bo()
411 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
419 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
426 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
433 * oa_buffer_check_unlocked - check for data and update tail ptr state
441 * pointer having a race with respect to what data is visible to the CPU.
458 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
459 int report_size = stream->oa_buffer.format_size; in oa_buffer_check_unlocked()
469 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
471 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
476 hw_tail &= ~(report_size - 1); in oa_buffer_check_unlocked()
480 if (hw_tail == stream->oa_buffer.aging_tail && in oa_buffer_check_unlocked()
481 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { in oa_buffer_check_unlocked()
486 stream->oa_buffer.tail = stream->oa_buffer.aging_tail; in oa_buffer_check_unlocked()
492 * anywhere between this head and stream->oa_buffer.tail. in oa_buffer_check_unlocked()
494 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
495 aged_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
497 hw_tail -= gtt_offset; in oa_buffer_check_unlocked()
512 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); in oa_buffer_check_unlocked()
517 tail = (tail - report_size) & (OA_BUFFER_SIZE - 1); in oa_buffer_check_unlocked()
521 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
526 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
527 stream->oa_buffer.aging_tail = gtt_offset + hw_tail; in oa_buffer_check_unlocked()
528 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
531 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, in oa_buffer_check_unlocked()
532 stream->oa_buffer.head - gtt_offset) >= report_size; in oa_buffer_check_unlocked()
534 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
540 * append_oa_status - Appends a status record to a userspace read() buffer.
541 * @stream: An i915-perf stream opened for OA metrics
562 if ((count - *offset) < header.size) in append_oa_status()
563 return -ENOSPC; in append_oa_status()
566 return -EFAULT; in append_oa_status()
574 * append_oa_sample - Copies single OA report into userspace read() buffer.
575 * @stream: An i915-perf stream opened for OA metrics
582 * properties when opening a stream, tracked as `stream->sample_flags`. This
596 int report_size = stream->oa_buffer.format_size; in append_oa_sample()
601 header.size = stream->sample_size; in append_oa_sample()
603 if ((count - *offset) < header.size) in append_oa_sample()
604 return -ENOSPC; in append_oa_sample()
608 return -EFAULT; in append_oa_sample()
612 return -EFAULT; in append_oa_sample()
620 * gen8_append_oa_reports - Copies all buffered OA reports into
622 * @stream: An i915-perf stream opened for OA metrics
627 * Notably any error condition resulting in a short read (-%ENOSPC or
628 * -%EFAULT) will be returned even though one or more records may
635 * and back-to-front you're not alone, but this follows the
645 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
646 int report_size = stream->oa_buffer.format_size; in gen8_append_oa_reports()
647 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
648 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
649 u32 mask = (OA_BUFFER_SIZE - 1); in gen8_append_oa_reports()
656 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
657 return -EIO; in gen8_append_oa_reports()
659 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
661 head = stream->oa_buffer.head; in gen8_append_oa_reports()
662 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
664 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
670 head -= gtt_offset; in gen8_append_oa_reports()
671 tail -= gtt_offset; in gen8_append_oa_reports()
680 if (drm_WARN_ONCE(&uncore->i915->drm, in gen8_append_oa_reports()
685 return -EIO; in gen8_append_oa_reports()
705 if (drm_WARN_ON(&uncore->i915->drm, in gen8_append_oa_reports()
706 (OA_BUFFER_SIZE - head) < report_size)) { in gen8_append_oa_reports()
707 drm_err(&uncore->i915->drm, in gen8_append_oa_reports()
708 "Spurious OA head ptr: non-integral report offset\n"); in gen8_append_oa_reports()
722 (GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
726 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
730 * invalid to be sure we avoid false-positive, single-context in gen8_append_oa_reports()
736 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && in gen8_append_oa_reports()
737 GRAPHICS_VER(stream->perf->i915) <= 11) in gen8_append_oa_reports()
743 * stop the counters from updating as system-wide / global in gen8_append_oa_reports()
747 * filtered on the cpu but it's not worth trying to in gen8_append_oa_reports()
751 * provide a side-band view of the real values. in gen8_append_oa_reports()
755 * needs be forwarded bookend context-switch reports so that it in gen8_append_oa_reports()
768 * switches since it's not-uncommon for periodic samples to in gen8_append_oa_reports()
771 if (!stream->perf->exclusive_stream->ctx || in gen8_append_oa_reports()
772 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
773 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
780 if (stream->perf->exclusive_stream->ctx && in gen8_append_oa_reports()
781 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
790 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
804 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
807 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
816 stream->oa_buffer.head = head; in gen8_append_oa_reports()
818 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
825 * gen8_oa_read - copy status records then buffered OA reports
826 * @stream: An i915-perf stream opened for OA metrics
849 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
854 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
855 return -EIO; in gen8_oa_read()
857 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
883 stream->period_exponent); in gen8_oa_read()
885 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
886 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
889 * Note: .oa_enable() is expected to re-init the oabuffer and in gen8_oa_read()
904 IS_GRAPHICS_VER(uncore->i915, 8, 11) ? in gen8_oa_read()
913 * gen7_append_oa_reports - Copies all buffered OA reports into
915 * @stream: An i915-perf stream opened for OA metrics
920 * Notably any error condition resulting in a short read (-%ENOSPC or
921 * -%EFAULT) will be returned even though one or more records may
928 * and back-to-front you're not alone, but this follows the
938 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
939 int report_size = stream->oa_buffer.format_size; in gen7_append_oa_reports()
940 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
941 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
942 u32 mask = (OA_BUFFER_SIZE - 1); in gen7_append_oa_reports()
949 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
950 return -EIO; in gen7_append_oa_reports()
952 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
954 head = stream->oa_buffer.head; in gen7_append_oa_reports()
955 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
957 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
962 head -= gtt_offset; in gen7_append_oa_reports()
963 tail -= gtt_offset; in gen7_append_oa_reports()
971 if (drm_WARN_ONCE(&uncore->i915->drm, in gen7_append_oa_reports()
976 return -EIO; in gen7_append_oa_reports()
993 if (drm_WARN_ON(&uncore->i915->drm, in gen7_append_oa_reports()
994 (OA_BUFFER_SIZE - head) < report_size)) { in gen7_append_oa_reports()
995 drm_err(&uncore->i915->drm, in gen7_append_oa_reports()
996 "Spurious OA head ptr: non-integral report offset\n"); in gen7_append_oa_reports()
1000 /* The report-ID field for periodic samples includes in gen7_append_oa_reports()
1007 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1024 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1034 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1036 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1043 * gen7_oa_read - copy status records then buffered OA reports
1044 * @stream: An i915-perf stream opened for OA metrics
1063 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1067 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1068 return -EIO; in gen7_oa_read()
1077 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1081 * - The status can be interpreted to mean that the buffer is in gen7_oa_read()
1083 * which will start to report a near-empty buffer after an in gen7_oa_read()
1088 * - Since it also implies the HW has started overwriting old in gen7_oa_read()
1093 * - In the future we may want to introduce a flight recorder in gen7_oa_read()
1106 stream->period_exponent); in gen7_oa_read()
1108 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1109 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1119 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1127 * i915_oa_wait_unlocked - handles blocking IO until OA data available
1128 * @stream: An i915-perf stream opened for OA metrics
1131 * for OA metrics. It waits until the hrtimer callback finds a non-empty
1135 * since any subsequent read handling will return -EAGAIN if there isn't
1143 if (!stream->periodic) in i915_oa_wait_unlocked()
1144 return -EIO; in i915_oa_wait_unlocked()
1146 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1151 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1152 * @stream: An i915-perf stream opened for OA metrics
1164 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1168 * i915_oa_read - just calls through to &i915_oa_ops->read
1169 * @stream: An i915-perf stream opened for OA metrics
1184 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1190 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1193 int err = -ENODEV; in oa_pin_context()
1196 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1214 if (err == -EDEADLK) { in oa_pin_context()
1224 stream->pinned_ctx = ce; in oa_pin_context()
1225 return stream->pinned_ctx; in oa_pin_context()
1229 * oa_get_render_ctx_id - determine and hold ctx hw id
1230 * @stream: An i915-perf stream opened for OA metrics
1246 switch (GRAPHICS_VER(ce->engine->i915)) { in oa_get_render_ctx_id()
1252 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1253 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1259 if (intel_engine_uses_guc(ce->engine)) { in oa_get_render_ctx_id()
1270 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1276 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1277 (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; in oa_get_render_ctx_id()
1279 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1280 (1U << GEN8_CTX_ID_WIDTH) - 1; in oa_get_render_ctx_id()
1281 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1287 if (GRAPHICS_VER_FULL(ce->engine->i915) >= IP_VER(12, 50)) { in oa_get_render_ctx_id()
1288 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1289 ((1U << XEHP_SW_CTX_ID_WIDTH) - 1) << in oa_get_render_ctx_id()
1290 (XEHP_SW_CTX_ID_SHIFT - 32); in oa_get_render_ctx_id()
1291 stream->specific_ctx_id = in oa_get_render_ctx_id()
1292 (XEHP_MAX_CONTEXT_HW_ID - 1) << in oa_get_render_ctx_id()
1293 (XEHP_SW_CTX_ID_SHIFT - 32); in oa_get_render_ctx_id()
1295 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1296 ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); in oa_get_render_ctx_id()
1299 * 0 - BITS_PER_LONG are used by other contexts in oa_get_render_ctx_id()
1302 stream->specific_ctx_id = in oa_get_render_ctx_id()
1303 (GEN12_MAX_CONTEXT_HW_ID - 1) << (GEN11_SW_CTX_ID_SHIFT - 32); in oa_get_render_ctx_id()
1308 MISSING_CASE(GRAPHICS_VER(ce->engine->i915)); in oa_get_render_ctx_id()
1311 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1313 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1315 stream->specific_ctx_id, in oa_get_render_ctx_id()
1316 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1322 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1323 * @stream: An i915-perf stream opened for OA metrics
1332 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1334 ce->tag = 0; /* recomputed on next submission after parking */ in oa_put_render_ctx_id()
1338 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1339 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1345 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1348 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1356 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1357 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1364 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1369 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1371 BUG_ON(stream != perf->exclusive_stream); in i915_oa_stream_destroy()
1379 WRITE_ONCE(perf->exclusive_stream, NULL); in i915_oa_stream_destroy()
1380 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1384 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1385 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1387 if (stream->ctx) in i915_oa_stream_destroy()
1393 if (perf->spurious_report_rs.missed) { in i915_oa_stream_destroy()
1395 perf->spurious_report_rs.missed); in i915_oa_stream_destroy()
1401 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1402 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1405 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1407 /* Pre-DevBDW: OABUFFER must be set with counters off, in gen7_init_oa_buffer()
1412 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1420 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1421 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1423 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1429 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1433 * first allocating), we may re-init the OA buffer, either in gen7_init_oa_buffer()
1434 * when re-enabling a stream or in error/reset paths. in gen7_init_oa_buffer()
1436 * The reason we clear the buffer for each re-init is for the in gen7_init_oa_buffer()
1438 * report-id field to make sure it's non-zero which relies on in gen7_init_oa_buffer()
1442 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1447 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1448 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1451 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1455 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1472 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1473 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1480 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1482 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1487 * first allocating), we may re-init the OA buffer, either in gen8_init_oa_buffer()
1488 * when re-enabling a stream or in error/reset paths. in gen8_init_oa_buffer()
1490 * The reason we clear the buffer for each re-init is for the in gen8_init_oa_buffer()
1492 * reason field to make sure it's non-zero which relies on in gen8_init_oa_buffer()
1496 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1501 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1502 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1505 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1510 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1526 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen12_init_oa_buffer()
1527 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1534 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1536 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1541 * first allocating), we may re-init the OA buffer, either in gen12_init_oa_buffer()
1542 * when re-enabling a stream or in error/reset paths. in gen12_init_oa_buffer()
1544 * The reason we clear the buffer for each re-init is for the in gen12_init_oa_buffer()
1546 * reason field to make sure it's non-zero which relies on in gen12_init_oa_buffer()
1550 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1551 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1556 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1561 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1562 return -ENODEV; in alloc_oa_buffer()
1567 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1569 drm_err(&i915->drm, "Failed to allocate OA buffer\n"); in alloc_oa_buffer()
1581 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1583 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1585 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1586 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1598 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1599 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1613 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1619 *cs++ = intel_gt_scratch_offset(stream->engine->gt, in save_restore_register()
1629 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1632 const u64 delay_ticks = 0xffffffffffffffff - in alloc_noa_wait()
1633 intel_gt_ns_to_clock_interval(stream->perf->i915->ggtt.vm.gt, in alloc_noa_wait()
1634 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1635 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1651 drm_err(&i915->drm, in alloc_noa_wait()
1699 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
1717 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
1737 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
1746 *cs++ = i915_ggtt_offset(vma) + (ts0 - batch) * 4; in alloc_noa_wait()
1751 * (((1 * << 64) - 1) - delay_ns) in alloc_noa_wait()
1774 *cs++ = MI_LOAD_REGISTER_REG | (3 - 2); in alloc_noa_wait()
1783 *cs++ = i915_ggtt_offset(vma) + (jump - batch) * 4; in alloc_noa_wait()
1798 GEM_BUG_ON(cs - batch > PAGE_SIZE / sizeof(*batch)); in alloc_noa_wait()
1803 stream->noa_wait = vma; in alloc_noa_wait()
1809 if (ret == -EDEADLK) { in alloc_noa_wait()
1829 n_regs - i, in write_cs_mi_lri()
1866 return ERR_PTR(-ENOMEM); in alloc_oa_config_buffer()
1868 config_length += num_lri_dwords(oa_config->mux_regs_len); in alloc_oa_config_buffer()
1869 config_length += num_lri_dwords(oa_config->b_counter_regs_len); in alloc_oa_config_buffer()
1870 config_length += num_lri_dwords(oa_config->flex_regs_len); in alloc_oa_config_buffer()
1874 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
1893 oa_config->mux_regs, in alloc_oa_config_buffer()
1894 oa_config->mux_regs_len); in alloc_oa_config_buffer()
1896 oa_config->b_counter_regs, in alloc_oa_config_buffer()
1897 oa_config->b_counter_regs_len); in alloc_oa_config_buffer()
1899 oa_config->flex_regs, in alloc_oa_config_buffer()
1900 oa_config->flex_regs_len); in alloc_oa_config_buffer()
1903 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
1906 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
1912 oa_bo->vma = i915_vma_instance(obj, in alloc_oa_config_buffer()
1913 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
1915 if (IS_ERR(oa_bo->vma)) { in alloc_oa_config_buffer()
1916 err = PTR_ERR(oa_bo->vma); in alloc_oa_config_buffer()
1920 oa_bo->oa_config = i915_oa_config_get(oa_config); in alloc_oa_config_buffer()
1921 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
1924 if (err == -EDEADLK) { in alloc_oa_config_buffer()
1950 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
1951 if (oa_bo->oa_config == oa_config && in get_oa_vma()
1952 memcmp(oa_bo->oa_config->uuid, in get_oa_vma()
1953 oa_config->uuid, in get_oa_vma()
1954 sizeof(oa_config->uuid)) == 0) in get_oa_vma()
1963 return i915_vma_get(oa_bo->vma); in get_oa_vma()
1983 err = i915_gem_object_lock(vma->obj, &ww); in emit_oa_config()
1991 intel_engine_pm_get(ce->engine); in emit_oa_config()
1993 intel_engine_pm_put(ce->engine); in emit_oa_config()
2011 err = i915_request_await_object(rq, vma->obj, 0); in emit_oa_config()
2017 err = rq->engine->emit_bb_start(rq, in emit_oa_config()
2018 vma->node.start, 0, in emit_oa_config()
2028 if (err == -EDEADLK) { in emit_oa_config()
2041 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2048 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2055 * unable to count the events from non-render clock domain. in hsw_enable_metric_set()
2057 * count the events from non-render domain. Unit level clock in hsw_enable_metric_set()
2066 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2072 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2096 for (i = 0; i < oa_config->flex_regs_len; i++) { in oa_config_flex_reg()
2097 if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio) in oa_config_flex_reg()
2098 return oa_config->flex_regs[i].value; in oa_config_flex_reg()
2107 * It's fine to put out-of-date values into these per-context registers
2114 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2115 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2126 u32 *reg_state = ce->lrc_reg_state; in gen8_update_reg_state_unlocked()
2130 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2131 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2136 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2157 offset = i915_ggtt_offset(ce->state) + LRC_STATE_OFFSET; in gen8_store_flex()
2160 *cs++ = offset + flex->offset * sizeof(u32); in gen8_store_flex()
2162 *cs++ = flex->value; in gen8_store_flex()
2163 } while (flex++, --count); in gen8_store_flex()
2185 *cs++ = i915_mmio_reg_offset(flex->reg); in gen8_load_flex()
2186 *cs++ = flex->value; in gen8_load_flex()
2187 } while (flex++, --count); in gen8_load_flex()
2201 rq = intel_engine_create_kernel_request(ce->engine); in gen8_modify_context()
2222 intel_engine_pm_get(ce->engine); in gen8_modify_self()
2224 intel_engine_pm_put(ce->engine); in gen8_modify_self()
2251 GEM_BUG_ON(ce == ce->engine->kernel_context); in gen8_configure_context()
2253 if (ce->engine->class != RENDER_CLASS) in gen8_configure_context()
2260 flex->value = intel_sseu_make_rpcs(ce->engine->gt, &ce->sseu); in gen8_configure_context()
2276 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2277 u32 format = stream->oa_buffer.format; in gen12_configure_oar_context()
2281 stream->perf->ctx_oactxctrl_offset + 1, in gen12_configure_oar_context()
2297 RING_CONTEXT_CONTROL(ce->engine->mmio_base), in gen12_configure_oar_context()
2321 * Manages updating the per-context aspects of the OA stream
2331 * won't automatically reload an out-of-date timer exponent even
2335 * - Ensure the currently running context's per-context OA state is
2337 * - Ensure that all existing contexts will have the correct per-context
2339 * - Ensure any new contexts will be initialized with the correct
2340 * per-context OA state.
2351 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2356 lockdep_assert_held(&stream->perf->lock); in oa_configure_all_contexts()
2361 * lite-restore). This means we can't safely update a context's image, in oa_configure_all_contexts()
2374 spin_lock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2375 list_for_each_entry_safe(ctx, cn, &i915->gem.contexts.list, link) { in oa_configure_all_contexts()
2376 if (!kref_get_unless_zero(&ctx->ref)) in oa_configure_all_contexts()
2379 spin_unlock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2387 spin_lock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2391 spin_unlock(&i915->gem.contexts.lock); in oa_configure_all_contexts()
2399 struct intel_context *ce = engine->kernel_context; in oa_configure_all_contexts()
2401 if (engine->class != RENDER_CLASS) in oa_configure_all_contexts()
2404 regs[0].value = intel_sseu_make_rpcs(engine->gt, &ce->sseu); in oa_configure_all_contexts()
2437 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2446 stream->perf->ctx_oactxctrl_offset + 1, in lrc_configure_all_contexts()
2460 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2461 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2476 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2477 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2492 * Currently none of the high-level metrics we have depend on knowing in gen8_enable_metric_set()
2503 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2519 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2526 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2534 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2535 struct i915_oa_config *oa_config = stream->oa_config; in gen12_enable_metric_set()
2536 bool periodic = stream->periodic; in gen12_enable_metric_set()
2537 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2570 if (stream->ctx) { in gen12_enable_metric_set()
2577 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2583 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2593 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2604 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2610 if (stream->ctx) in gen12_disable_metric_set()
2619 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
2620 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2621 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2622 bool periodic = stream->periodic; in gen7_oa_enable()
2623 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2624 u32 report_format = stream->oa_buffer.format; in gen7_oa_enable()
2649 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
2650 u32 report_format = stream->oa_buffer.format; in gen8_oa_enable()
2665 * filtering and instead filter on the cpu based on the context-id in gen8_oa_enable()
2675 struct intel_uncore *uncore = stream->uncore; in gen12_oa_enable()
2676 u32 report_format = stream->oa_buffer.format; in gen12_oa_enable()
2682 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
2693 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
2703 stream->pollin = false; in i915_oa_stream_enable()
2705 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
2707 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
2708 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
2709 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
2715 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
2721 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
2727 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
2733 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
2739 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
2746 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2754 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2759 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
2768 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
2770 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
2771 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
2790 return -ENOMEM; in i915_perf_stream_enable_sync()
2792 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
2804 const struct sseu_dev_info *devinfo_sseu = &engine->gt->info.sseu; in get_default_sseu_config()
2808 if (GRAPHICS_VER(engine->i915) == 11) { in get_default_sseu_config()
2811 * we select - just turn off low bits in the amount of half of in get_default_sseu_config()
2814 out_sseu->subslice_mask = in get_default_sseu_config()
2815 ~(~0 << (hweight8(out_sseu->subslice_mask) / 2)); in get_default_sseu_config()
2816 out_sseu->slice_mask = 0x1; in get_default_sseu_config()
2825 if (drm_sseu->engine.engine_class != engine->uabi_class || in get_sseu_config()
2826 drm_sseu->engine.engine_instance != engine->uabi_instance) in get_sseu_config()
2827 return -EINVAL; in get_sseu_config()
2829 return i915_gem_user_to_context_sseu(engine->gt, drm_sseu, out_sseu); in get_sseu_config()
2833 * i915_oa_stream_init - validate combined props for OA stream and init
2854 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
2855 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
2859 if (!props->engine) { in i915_oa_stream_init()
2861 return -EINVAL; in i915_oa_stream_init()
2869 if (!perf->metrics_kobj) { in i915_oa_stream_init()
2871 return -EINVAL; in i915_oa_stream_init()
2874 if (!(props->sample_flags & SAMPLE_OA_REPORT) && in i915_oa_stream_init()
2875 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
2877 return -EINVAL; in i915_oa_stream_init()
2880 if (!perf->ops.enable_metric_set) { in i915_oa_stream_init()
2882 return -ENODEV; in i915_oa_stream_init()
2890 if (perf->exclusive_stream) { in i915_oa_stream_init()
2892 return -EBUSY; in i915_oa_stream_init()
2895 if (!props->oa_format) { in i915_oa_stream_init()
2897 return -EINVAL; in i915_oa_stream_init()
2900 stream->engine = props->engine; in i915_oa_stream_init()
2901 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
2903 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
2905 format_size = perf->oa_formats[props->oa_format].size; in i915_oa_stream_init()
2907 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
2908 stream->sample_size += format_size; in i915_oa_stream_init()
2910 stream->oa_buffer.format_size = format_size; in i915_oa_stream_init()
2911 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0)) in i915_oa_stream_init()
2912 return -EINVAL; in i915_oa_stream_init()
2914 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
2916 stream->oa_buffer.format = in i915_oa_stream_init()
2917 perf->oa_formats[props->oa_format].format; in i915_oa_stream_init()
2919 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
2920 if (stream->periodic) in i915_oa_stream_init()
2921 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2923 if (stream->ctx) { in i915_oa_stream_init()
2937 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
2938 if (!stream->oa_config) { in i915_oa_stream_init()
2939 DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set); in i915_oa_stream_init()
2940 ret = -EINVAL; in i915_oa_stream_init()
2944 /* PRM - observability performance counters: in i915_oa_stream_init()
2956 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
2957 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2963 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
2965 perf->sseu = props->sseu; in i915_oa_stream_init()
2966 WRITE_ONCE(perf->exclusive_stream, stream); in i915_oa_stream_init()
2975 stream->oa_config->uuid); in i915_oa_stream_init()
2977 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
2979 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
2980 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
2981 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
2986 WRITE_ONCE(perf->exclusive_stream, NULL); in i915_oa_stream_init()
2987 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
2994 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2995 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
3001 if (stream->ctx) in i915_oa_stream_init()
3012 if (engine->class != RENDER_CLASS) in i915_oa_init_reg_state()
3016 stream = READ_ONCE(engine->i915->perf.exclusive_stream); in i915_oa_init_reg_state()
3017 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3022 * i915_perf_read - handles read() FOP for i915 perf stream FDs
3030 * &i915_perf_stream_ops->read but to save having stream implementations (of
3044 struct i915_perf_stream *stream = file->private_data; in i915_perf_read()
3045 struct i915_perf *perf = stream->perf; in i915_perf_read()
3053 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3054 return -EIO; in i915_perf_read()
3056 if (!(file->f_flags & O_NONBLOCK)) { in i915_perf_read()
3058 * stream->ops->wait_unlocked. in i915_perf_read()
3065 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3069 mutex_lock(&perf->lock); in i915_perf_read()
3070 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3071 mutex_unlock(&perf->lock); in i915_perf_read()
3074 mutex_lock(&perf->lock); in i915_perf_read()
3075 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3076 mutex_unlock(&perf->lock); in i915_perf_read()
3083 * and read() returning -EAGAIN. Clearing the oa.pollin state here in i915_perf_read()
3086 * The exception to this is if ops->read() returned -ENOSPC which means in i915_perf_read()
3090 if (ret != -ENOSPC) in i915_perf_read()
3091 stream->pollin = false; in i915_perf_read()
3093 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, ... */ in i915_perf_read()
3094 return offset ?: (ret ?: -EAGAIN); in i915_perf_read()
3103 stream->pollin = true; in oa_poll_check_timer_cb()
3104 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3108 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3114 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3120 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
3123 * Note: The &perf->lock mutex has been taken to serialize
3124 * with any non-file-operation driver hooks.
3134 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3142 if (stream->pollin) in i915_perf_poll_locked()
3149 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3163 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll()
3164 struct i915_perf *perf = stream->perf; in i915_perf_poll()
3167 mutex_lock(&perf->lock); in i915_perf_poll()
3169 mutex_unlock(&perf->lock); in i915_perf_poll()
3175 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
3186 if (stream->enabled) in i915_perf_enable_locked()
3189 /* Allow stream->ops->enable() to refer to this */ in i915_perf_enable_locked()
3190 stream->enabled = true; in i915_perf_enable_locked()
3192 if (stream->ops->enable) in i915_perf_enable_locked()
3193 stream->ops->enable(stream); in i915_perf_enable_locked()
3195 if (stream->hold_preemption) in i915_perf_enable_locked()
3196 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3200 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
3205 * The intention is that disabling an re-enabling a stream will ideally be
3206 * cheaper than destroying and re-opening a stream with the same configuration,
3208 * must be retained between disabling and re-enabling a stream.
3211 * to attempt to read from the stream (-EIO).
3215 if (!stream->enabled) in i915_perf_disable_locked()
3218 /* Allow stream->ops->disable() to refer to this */ in i915_perf_disable_locked()
3219 stream->enabled = false; in i915_perf_disable_locked()
3221 if (stream->hold_preemption) in i915_perf_disable_locked()
3222 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3224 if (stream->ops->disable) in i915_perf_disable_locked()
3225 stream->ops->disable(stream); in i915_perf_disable_locked()
3232 long ret = stream->oa_config->id; in i915_perf_config_locked()
3234 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3236 return -EINVAL; in i915_perf_config_locked()
3238 if (config != stream->oa_config) { in i915_perf_config_locked()
3252 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3263 * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3268 * Note: The &perf->lock mutex has been taken to serialize
3269 * with any non-file-operation driver hooks.
3271 * Returns: zero on success or a negative error code. Returns -EINVAL for
3289 return -EINVAL; in i915_perf_ioctl_locked()
3293 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3300 * Returns: zero on success or a negative error code. Returns -EINVAL for
3307 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl()
3308 struct i915_perf *perf = stream->perf; in i915_perf_ioctl()
3311 mutex_lock(&perf->lock); in i915_perf_ioctl()
3313 mutex_unlock(&perf->lock); in i915_perf_ioctl()
3319 * i915_perf_destroy_locked - destroy an i915 perf stream
3325 * Note: The &perf->lock mutex has been taken to serialize
3326 * with any non-file-operation driver hooks.
3330 if (stream->enabled) in i915_perf_destroy_locked()
3333 if (stream->ops->destroy) in i915_perf_destroy_locked()
3334 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3336 if (stream->ctx) in i915_perf_destroy_locked()
3337 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3343 * i915_perf_release - handles userspace close() of a stream file
3355 struct i915_perf_stream *stream = file->private_data; in i915_perf_release()
3356 struct i915_perf *perf = stream->perf; in i915_perf_release()
3358 mutex_lock(&perf->lock); in i915_perf_release()
3360 mutex_unlock(&perf->lock); in i915_perf_release()
3363 drm_dev_put(&perf->i915->drm); in i915_perf_release()
3384 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3393 * behalf of i915_perf_open_ioctl() with the &perf->lock mutex
3394 * taken to serialize with any non-file-operation driver hooks.
3420 if (props->single_context) { in i915_perf_open_ioctl_locked()
3421 u32 ctx_handle = props->ctx_handle; in i915_perf_open_ioctl_locked()
3422 struct drm_i915_file_private *file_priv = file->driver_priv; in i915_perf_open_ioctl_locked()
3437 * non-privileged client. in i915_perf_open_ioctl_locked()
3439 * For Gen8->11 the OA unit no longer supports clock gating off for a in i915_perf_open_ioctl_locked()
3441 * from updating as system-wide / global values. Even though we can in i915_perf_open_ioctl_locked()
3452 if (IS_HASWELL(perf->i915) && specific_ctx) in i915_perf_open_ioctl_locked()
3454 else if (GRAPHICS_VER(perf->i915) == 12 && specific_ctx && in i915_perf_open_ioctl_locked()
3455 (props->sample_flags & SAMPLE_OA_REPORT) == 0) in i915_perf_open_ioctl_locked()
3458 if (props->hold_preemption) { in i915_perf_open_ioctl_locked()
3459 if (!props->single_context) { in i915_perf_open_ioctl_locked()
3461 ret = -EINVAL; in i915_perf_open_ioctl_locked()
3470 if (props->has_sseu) in i915_perf_open_ioctl_locked()
3473 get_default_sseu_config(&props->sseu, props->engine); in i915_perf_open_ioctl_locked()
3483 ret = -EACCES; in i915_perf_open_ioctl_locked()
3489 ret = -ENOMEM; in i915_perf_open_ioctl_locked()
3493 stream->perf = perf; in i915_perf_open_ioctl_locked()
3494 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3495 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3501 /* we avoid simply assigning stream->sample_flags = props->sample_flags in i915_perf_open_ioctl_locked()
3505 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3506 ret = -ENODEV; in i915_perf_open_ioctl_locked()
3510 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) in i915_perf_open_ioctl_locked()
3512 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) in i915_perf_open_ioctl_locked()
3521 if (!(param->flags & I915_PERF_FLAG_DISABLED)) in i915_perf_open_ioctl_locked()
3527 drm_dev_get(&perf->i915->drm); in i915_perf_open_ioctl_locked()
3532 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3533 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3545 return intel_gt_clock_interval_to_ns(perf->i915->ggtt.vm.gt, in oa_exponent_to_ns()
3552 return test_bit(format, perf->format_mask); in oa_format_valid()
3558 __set_bit(format, perf->format_mask); in oa_format_add()
3562 * read_properties_unlocked - validate + copy userspace stream open properties
3586 props->poll_oa_period = DEFAULT_POLL_PERIOD_NS; in read_properties_unlocked()
3590 return -EINVAL; in read_properties_unlocked()
3593 /* At the moment we only support using i915-perf on the RCS. */ in read_properties_unlocked()
3594 props->engine = intel_engine_lookup_user(perf->i915, in read_properties_unlocked()
3597 if (!props->engine) { in read_properties_unlocked()
3598 DRM_DEBUG("No RENDER-capable engines\n"); in read_properties_unlocked()
3599 return -EINVAL; in read_properties_unlocked()
3610 return -EINVAL; in read_properties_unlocked()
3627 return -EINVAL; in read_properties_unlocked()
3632 props->single_context = 1; in read_properties_unlocked()
3633 props->ctx_handle = value; in read_properties_unlocked()
3637 props->sample_flags |= SAMPLE_OA_REPORT; in read_properties_unlocked()
3642 return -EINVAL; in read_properties_unlocked()
3644 props->metrics_set = value; in read_properties_unlocked()
3648 DRM_DEBUG("Out-of-range OA report format %llu\n", in read_properties_unlocked()
3650 return -EINVAL; in read_properties_unlocked()
3655 return -EINVAL; in read_properties_unlocked()
3657 props->oa_format = value; in read_properties_unlocked()
3663 return -EINVAL; in read_properties_unlocked()
3691 return -EACCES; in read_properties_unlocked()
3694 props->oa_periodic = true; in read_properties_unlocked()
3695 props->oa_period_exponent = value; in read_properties_unlocked()
3698 props->hold_preemption = !!value; in read_properties_unlocked()
3707 return -EFAULT; in read_properties_unlocked()
3710 ret = get_sseu_config(&props->sseu, props->engine, &user_sseu); in read_properties_unlocked()
3715 props->has_sseu = true; in read_properties_unlocked()
3722 return -EINVAL; in read_properties_unlocked()
3724 props->poll_oa_period = value; in read_properties_unlocked()
3728 return -EINVAL; in read_properties_unlocked()
3738 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
3748 * i915-perf stream is expected to be a suitable interface for other forms of
3755 * i915_perf_open_ioctl_locked() after taking the &perf->lock
3756 * mutex for serializing with any non-file-operation driver hooks.
3764 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_open_ioctl()
3770 if (!perf->i915) { in i915_perf_open_ioctl()
3772 return -ENOTSUPP; in i915_perf_open_ioctl()
3778 if (param->flags & ~known_open_flags) { in i915_perf_open_ioctl()
3780 return -EINVAL; in i915_perf_open_ioctl()
3784 u64_to_user_ptr(param->properties_ptr), in i915_perf_open_ioctl()
3785 param->num_properties, in i915_perf_open_ioctl()
3790 mutex_lock(&perf->lock); in i915_perf_open_ioctl()
3792 mutex_unlock(&perf->lock); in i915_perf_open_ioctl()
3798 * i915_perf_register - exposes i915-perf to userspace
3803 * used to open an i915-perf stream.
3807 struct i915_perf *perf = &i915->perf; in i915_perf_register()
3809 if (!perf->i915) in i915_perf_register()
3816 mutex_lock(&perf->lock); in i915_perf_register()
3818 perf->metrics_kobj = in i915_perf_register()
3820 &i915->drm.primary->kdev->kobj); in i915_perf_register()
3822 mutex_unlock(&perf->lock); in i915_perf_register()
3826 * i915_perf_unregister - hide i915-perf from userspace
3829 * i915-perf state cleanup is split up into an 'unregister' and
3836 struct i915_perf *perf = &i915->perf; in i915_perf_unregister()
3838 if (!perf->metrics_kobj) in i915_perf_unregister()
3841 kobject_put(perf->metrics_kobj); in i915_perf_unregister()
3842 perf->metrics_kobj = NULL; in i915_perf_unregister()
3975 return ERR_PTR(-EINVAL); in alloc_oa_regs()
3979 return ERR_PTR(-ENOMEM); in alloc_oa_regs()
3990 err = -EINVAL; in alloc_oa_regs()
4018 return sprintf(buf, "%d\n", oa_config->id); in show_dynamic_id()
4024 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); in create_dynamic_oa_sysfs_entry()
4025 oa_config->sysfs_metric_id.attr.name = "id"; in create_dynamic_oa_sysfs_entry()
4026 oa_config->sysfs_metric_id.attr.mode = S_IRUGO; in create_dynamic_oa_sysfs_entry()
4027 oa_config->sysfs_metric_id.show = show_dynamic_id; in create_dynamic_oa_sysfs_entry()
4028 oa_config->sysfs_metric_id.store = NULL; in create_dynamic_oa_sysfs_entry()
4030 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; in create_dynamic_oa_sysfs_entry()
4031 oa_config->attrs[1] = NULL; in create_dynamic_oa_sysfs_entry()
4033 oa_config->sysfs_metric.name = oa_config->uuid; in create_dynamic_oa_sysfs_entry()
4034 oa_config->sysfs_metric.attrs = oa_config->attrs; in create_dynamic_oa_sysfs_entry()
4036 return sysfs_create_group(perf->metrics_kobj, in create_dynamic_oa_sysfs_entry()
4037 &oa_config->sysfs_metric); in create_dynamic_oa_sysfs_entry()
4041 * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
4056 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_add_config_ioctl()
4062 if (!perf->i915) { in i915_perf_add_config_ioctl()
4064 return -ENOTSUPP; in i915_perf_add_config_ioctl()
4067 if (!perf->metrics_kobj) { in i915_perf_add_config_ioctl()
4069 return -EINVAL; in i915_perf_add_config_ioctl()
4074 return -EACCES; in i915_perf_add_config_ioctl()
4077 if ((!args->mux_regs_ptr || !args->n_mux_regs) && in i915_perf_add_config_ioctl()
4078 (!args->boolean_regs_ptr || !args->n_boolean_regs) && in i915_perf_add_config_ioctl()
4079 (!args->flex_regs_ptr || !args->n_flex_regs)) { in i915_perf_add_config_ioctl()
4081 return -EINVAL; in i915_perf_add_config_ioctl()
4087 return -ENOMEM; in i915_perf_add_config_ioctl()
4090 oa_config->perf = perf; in i915_perf_add_config_ioctl()
4091 kref_init(&oa_config->ref); in i915_perf_add_config_ioctl()
4093 if (!uuid_is_valid(args->uuid)) { in i915_perf_add_config_ioctl()
4095 err = -EINVAL; in i915_perf_add_config_ioctl()
4099 /* Last character in oa_config->uuid will be 0 because oa_config is in i915_perf_add_config_ioctl()
4102 memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid)); in i915_perf_add_config_ioctl()
4104 oa_config->mux_regs_len = args->n_mux_regs; in i915_perf_add_config_ioctl()
4106 perf->ops.is_valid_mux_reg, in i915_perf_add_config_ioctl()
4107 u64_to_user_ptr(args->mux_regs_ptr), in i915_perf_add_config_ioctl()
4108 args->n_mux_regs); in i915_perf_add_config_ioctl()
4115 oa_config->mux_regs = regs; in i915_perf_add_config_ioctl()
4117 oa_config->b_counter_regs_len = args->n_boolean_regs; in i915_perf_add_config_ioctl()
4119 perf->ops.is_valid_b_counter_reg, in i915_perf_add_config_ioctl()
4120 u64_to_user_ptr(args->boolean_regs_ptr), in i915_perf_add_config_ioctl()
4121 args->n_boolean_regs); in i915_perf_add_config_ioctl()
4128 oa_config->b_counter_regs = regs; in i915_perf_add_config_ioctl()
4130 if (GRAPHICS_VER(perf->i915) < 8) { in i915_perf_add_config_ioctl()
4131 if (args->n_flex_regs != 0) { in i915_perf_add_config_ioctl()
4132 err = -EINVAL; in i915_perf_add_config_ioctl()
4136 oa_config->flex_regs_len = args->n_flex_regs; in i915_perf_add_config_ioctl()
4138 perf->ops.is_valid_flex_reg, in i915_perf_add_config_ioctl()
4139 u64_to_user_ptr(args->flex_regs_ptr), in i915_perf_add_config_ioctl()
4140 args->n_flex_regs); in i915_perf_add_config_ioctl()
4147 oa_config->flex_regs = regs; in i915_perf_add_config_ioctl()
4150 err = mutex_lock_interruptible(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4157 idr_for_each_entry(&perf->metrics_idr, tmp, id) { in i915_perf_add_config_ioctl()
4158 if (!strcmp(tmp->uuid, oa_config->uuid)) { in i915_perf_add_config_ioctl()
4160 err = -EADDRINUSE; in i915_perf_add_config_ioctl()
4172 oa_config->id = idr_alloc(&perf->metrics_idr, in i915_perf_add_config_ioctl()
4175 if (oa_config->id < 0) { in i915_perf_add_config_ioctl()
4177 err = oa_config->id; in i915_perf_add_config_ioctl()
4181 mutex_unlock(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4183 DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_add_config_ioctl()
4185 return oa_config->id; in i915_perf_add_config_ioctl()
4188 mutex_unlock(&perf->metrics_lock); in i915_perf_add_config_ioctl()
4196 * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
4209 struct i915_perf *perf = &to_i915(dev)->perf; in i915_perf_remove_config_ioctl()
4214 if (!perf->i915) { in i915_perf_remove_config_ioctl()
4216 return -ENOTSUPP; in i915_perf_remove_config_ioctl()
4221 return -EACCES; in i915_perf_remove_config_ioctl()
4224 ret = mutex_lock_interruptible(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4228 oa_config = idr_find(&perf->metrics_idr, *arg); in i915_perf_remove_config_ioctl()
4231 ret = -ENOENT; in i915_perf_remove_config_ioctl()
4235 GEM_BUG_ON(*arg != oa_config->id); in i915_perf_remove_config_ioctl()
4237 sysfs_remove_group(perf->metrics_kobj, &oa_config->sysfs_metric); in i915_perf_remove_config_ioctl()
4239 idr_remove(&perf->metrics_idr, *arg); in i915_perf_remove_config_ioctl()
4241 mutex_unlock(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4243 DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_remove_config_ioctl()
4250 mutex_unlock(&perf->metrics_lock); in i915_perf_remove_config_ioctl()
4298 struct drm_i915_private *i915 = perf->i915; in oa_init_supported_formats()
4299 enum intel_platform platform = INTEL_INFO(i915)->platform; in oa_init_supported_formats()
4341 * i915_perf_init - initialize i915-perf state on module bind
4344 * Initializes i915-perf state without exposing anything to userspace.
4346 * Note: i915-perf initialization is split into an 'init' and 'register'
4351 struct i915_perf *perf = &i915->perf; in i915_perf_init()
4355 perf->oa_formats = oa_formats; in i915_perf_init()
4357 perf->ops.is_valid_b_counter_reg = gen7_is_valid_b_counter_addr; in i915_perf_init()
4358 perf->ops.is_valid_mux_reg = hsw_is_valid_mux_addr; in i915_perf_init()
4359 perf->ops.is_valid_flex_reg = NULL; in i915_perf_init()
4360 perf->ops.enable_metric_set = hsw_enable_metric_set; in i915_perf_init()
4361 perf->ops.disable_metric_set = hsw_disable_metric_set; in i915_perf_init()
4362 perf->ops.oa_enable = gen7_oa_enable; in i915_perf_init()
4363 perf->ops.oa_disable = gen7_oa_disable; in i915_perf_init()
4364 perf->ops.read = gen7_oa_read; in i915_perf_init()
4365 perf->ops.oa_hw_tail_read = gen7_oa_hw_tail_read; in i915_perf_init()
4373 perf->ops.read = gen8_oa_read; in i915_perf_init()
4376 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
4378 perf->ops.is_valid_mux_reg = in i915_perf_init()
4380 perf->ops.is_valid_flex_reg = in i915_perf_init()
4384 perf->ops.is_valid_mux_reg = in i915_perf_init()
4388 perf->ops.oa_enable = gen8_oa_enable; in i915_perf_init()
4389 perf->ops.oa_disable = gen8_oa_disable; in i915_perf_init()
4390 perf->ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
4391 perf->ops.disable_metric_set = gen8_disable_metric_set; in i915_perf_init()
4392 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
4395 perf->ctx_oactxctrl_offset = 0x120; in i915_perf_init()
4396 perf->ctx_flexeu0_offset = 0x2ce; in i915_perf_init()
4398 perf->gen8_valid_ctx_bit = BIT(25); in i915_perf_init()
4400 perf->ctx_oactxctrl_offset = 0x128; in i915_perf_init()
4401 perf->ctx_flexeu0_offset = 0x3de; in i915_perf_init()
4403 perf->gen8_valid_ctx_bit = BIT(16); in i915_perf_init()
4406 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
4408 perf->ops.is_valid_mux_reg = in i915_perf_init()
4410 perf->ops.is_valid_flex_reg = in i915_perf_init()
4413 perf->ops.oa_enable = gen8_oa_enable; in i915_perf_init()
4414 perf->ops.oa_disable = gen8_oa_disable; in i915_perf_init()
4415 perf->ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
4416 perf->ops.disable_metric_set = gen11_disable_metric_set; in i915_perf_init()
4417 perf->ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
4419 perf->ctx_oactxctrl_offset = 0x124; in i915_perf_init()
4420 perf->ctx_flexeu0_offset = 0x78e; in i915_perf_init()
4422 perf->gen8_valid_ctx_bit = BIT(16); in i915_perf_init()
4424 perf->ops.is_valid_b_counter_reg = in i915_perf_init()
4426 perf->ops.is_valid_mux_reg = in i915_perf_init()
4428 perf->ops.is_valid_flex_reg = in i915_perf_init()
4431 perf->ops.oa_enable = gen12_oa_enable; in i915_perf_init()
4432 perf->ops.oa_disable = gen12_oa_disable; in i915_perf_init()
4433 perf->ops.enable_metric_set = gen12_enable_metric_set; in i915_perf_init()
4434 perf->ops.disable_metric_set = gen12_disable_metric_set; in i915_perf_init()
4435 perf->ops.oa_hw_tail_read = gen12_oa_hw_tail_read; in i915_perf_init()
4437 perf->ctx_flexeu0_offset = 0; in i915_perf_init()
4438 perf->ctx_oactxctrl_offset = 0x144; in i915_perf_init()
4442 if (perf->ops.enable_metric_set) { in i915_perf_init()
4443 mutex_init(&perf->lock); in i915_perf_init()
4446 oa_sample_rate_hard_limit = i915->gt.clock_frequency / 2; in i915_perf_init()
4448 mutex_init(&perf->metrics_lock); in i915_perf_init()
4449 idr_init_base(&perf->metrics_idr, 1); in i915_perf_init()
4461 ratelimit_state_init(&perf->spurious_report_rs, 5 * HZ, 10); in i915_perf_init()
4466 ratelimit_set_flags(&perf->spurious_report_rs, in i915_perf_init()
4469 ratelimit_state_init(&perf->tail_pointer_race, in i915_perf_init()
4471 ratelimit_set_flags(&perf->tail_pointer_race, in i915_perf_init()
4474 atomic64_set(&perf->noa_programming_delay, in i915_perf_init()
4477 perf->i915 = i915; in i915_perf_init()
4501 * i915_perf_fini - Counter part to i915_perf_init()
4506 struct i915_perf *perf = &i915->perf; in i915_perf_fini()
4508 if (!perf->i915) in i915_perf_fini()
4511 idr_for_each(&perf->metrics_idr, destroy_config, perf); in i915_perf_fini()
4512 idr_destroy(&perf->metrics_idr); in i915_perf_fini()
4514 memset(&perf->ops, 0, sizeof(perf->ops)); in i915_perf_fini()
4515 perf->i915 = NULL; in i915_perf_fini()
4519 * i915_perf_ioctl_version - Version of the i915-perf subsystem