Lines Matching full:stream
36 * descriptor representing a stream of GPU metrics which can then be read() as
37 * a stream of sample records.
56 * i915 perf file descriptors represent a "stream" instead of an "event"; where
57 * a perf event primarily corresponds to a single 64bit value, while a stream
61 * of related counters. Samples for an i915 perf stream capturing OA metrics
64 * selected by the user opening the stream. Perf has support for grouping
68 * i915 perf stream configurations are provided as an array of u64 (key,value)
322 * struct perf_open_properties - for validated properties given to open a stream
341 * to open a stream of metrics the configuration is built up in the structure
409 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream) in gen12_oa_hw_tail_read() argument
411 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
417 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream) in gen8_oa_hw_tail_read() argument
419 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
424 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) in gen7_oa_hw_tail_read() argument
426 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
434 * @stream: i915 stream instance
451 * only called while the stream is enabled, while the global OA configuration
456 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) in oa_buffer_check_unlocked() argument
458 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
459 int report_size = stream->oa_buffer.format_size; in oa_buffer_check_unlocked()
469 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
471 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
480 if (hw_tail == stream->oa_buffer.aging_tail && in oa_buffer_check_unlocked()
481 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { in oa_buffer_check_unlocked()
486 stream->oa_buffer.tail = stream->oa_buffer.aging_tail; in oa_buffer_check_unlocked()
492 * anywhere between this head and stream->oa_buffer.tail. in oa_buffer_check_unlocked()
494 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
495 aged_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
500 /* Walk the stream backward until we find a report with dword 0 in oa_buffer_check_unlocked()
512 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); in oa_buffer_check_unlocked()
521 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
526 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
527 stream->oa_buffer.aging_tail = gtt_offset + hw_tail; in oa_buffer_check_unlocked()
528 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
531 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, in oa_buffer_check_unlocked()
532 stream->oa_buffer.head - gtt_offset) >= report_size; in oa_buffer_check_unlocked()
534 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
541 * @stream: An i915-perf stream opened for OA metrics
554 static int append_oa_status(struct i915_perf_stream *stream, in append_oa_status() argument
575 * @stream: An i915-perf stream opened for OA metrics
582 * properties when opening a stream, tracked as `stream->sample_flags`. This
590 static int append_oa_sample(struct i915_perf_stream *stream, in append_oa_sample() argument
596 int report_size = stream->oa_buffer.format_size; in append_oa_sample()
601 header.size = stream->sample_size; in append_oa_sample()
622 * @stream: An i915-perf stream opened for OA metrics
640 static int gen8_append_oa_reports(struct i915_perf_stream *stream, in gen8_append_oa_reports() argument
645 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
646 int report_size = stream->oa_buffer.format_size; in gen8_append_oa_reports()
647 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
648 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
656 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
659 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
661 head = stream->oa_buffer.head; in gen8_append_oa_reports()
662 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
664 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
722 (GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
726 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
736 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && in gen8_append_oa_reports()
737 GRAPHICS_VER(stream->perf->i915) <= 11) in gen8_append_oa_reports()
771 if (!stream->perf->exclusive_stream->ctx || in gen8_append_oa_reports()
772 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
773 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
780 if (stream->perf->exclusive_stream->ctx && in gen8_append_oa_reports()
781 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
785 ret = append_oa_sample(stream, buf, count, offset, in gen8_append_oa_reports()
790 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
804 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
807 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
816 stream->oa_buffer.head = head; in gen8_append_oa_reports()
818 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
826 * @stream: An i915-perf stream opened for OA metrics
844 static int gen8_oa_read(struct i915_perf_stream *stream, in gen8_oa_read() argument
849 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
854 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
857 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
877 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
883 stream->period_exponent); in gen8_oa_read()
885 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
886 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
896 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
909 return gen8_append_oa_reports(stream, buf, count, offset); in gen8_oa_read()
915 * @stream: An i915-perf stream opened for OA metrics
933 static int gen7_append_oa_reports(struct i915_perf_stream *stream, in gen7_append_oa_reports() argument
938 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
939 int report_size = stream->oa_buffer.format_size; in gen7_append_oa_reports()
940 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
941 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
949 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
952 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
954 head = stream->oa_buffer.head; in gen7_append_oa_reports()
955 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
957 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1007 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1012 ret = append_oa_sample(stream, buf, count, offset, report); in gen7_append_oa_reports()
1024 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1034 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1036 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1044 * @stream: An i915-perf stream opened for OA metrics
1058 static int gen7_oa_read(struct i915_perf_stream *stream, in gen7_oa_read() argument
1063 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1067 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1077 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1100 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1106 stream->period_exponent); in gen7_oa_read()
1108 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1109 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1115 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1119 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1123 return gen7_append_oa_reports(stream, buf, count, offset); in gen7_oa_read()
1128 * @stream: An i915-perf stream opened for OA metrics
1130 * Called when userspace tries to read() from a blocking stream FD opened
1140 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) in i915_oa_wait_unlocked() argument
1143 if (!stream->periodic) in i915_oa_wait_unlocked()
1146 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1147 oa_buffer_check_unlocked(stream)); in i915_oa_wait_unlocked()
1151 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1152 * @stream: An i915-perf stream opened for OA metrics
1153 * @file: An i915 perf stream file
1156 * For handling userspace polling on an i915 perf stream opened for OA metrics,
1160 static void i915_oa_poll_wait(struct i915_perf_stream *stream, in i915_oa_poll_wait() argument
1164 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1169 * @stream: An i915-perf stream opened for OA metrics
1179 static int i915_oa_read(struct i915_perf_stream *stream, in i915_oa_read() argument
1184 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1187 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) in oa_pin_context() argument
1190 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1196 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1224 stream->pinned_ctx = ce; in oa_pin_context()
1225 return stream->pinned_ctx; in oa_pin_context()
1230 * @stream: An i915-perf stream opened for OA metrics
1233 * lifetime of the stream. This ensures that we don't have to worry about
1238 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) in oa_get_render_ctx_id() argument
1242 ce = oa_pin_context(stream); in oa_get_render_ctx_id()
1252 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1253 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1270 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1276 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1279 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1281 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1288 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1291 stream->specific_ctx_id = in oa_get_render_ctx_id()
1295 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1302 stream->specific_ctx_id = in oa_get_render_ctx_id()
1311 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1313 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1315 stream->specific_ctx_id, in oa_get_render_ctx_id()
1316 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1323 * @stream: An i915-perf stream opened for OA metrics
1326 * for the lifetime of the stream, then that can be undone here.
1328 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) in oa_put_render_ctx_id() argument
1332 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1338 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1339 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1343 free_oa_buffer(struct i915_perf_stream *stream) in free_oa_buffer() argument
1345 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1348 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1352 free_oa_configs(struct i915_perf_stream *stream) in free_oa_configs() argument
1356 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1357 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1362 free_noa_wait(struct i915_perf_stream *stream) in free_noa_wait() argument
1364 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1367 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) in i915_oa_stream_destroy() argument
1369 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1371 BUG_ON(stream != perf->exclusive_stream); in i915_oa_stream_destroy()
1380 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1382 free_oa_buffer(stream); in i915_oa_stream_destroy()
1384 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1385 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1387 if (stream->ctx) in i915_oa_stream_destroy()
1388 oa_put_render_ctx_id(stream); in i915_oa_stream_destroy()
1390 free_oa_configs(stream); in i915_oa_stream_destroy()
1391 free_noa_wait(stream); in i915_oa_stream_destroy()
1399 static void gen7_init_oa_buffer(struct i915_perf_stream *stream) in gen7_init_oa_buffer() argument
1401 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1402 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1405 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1412 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1420 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1421 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1423 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1429 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1434 * when re-enabling a stream or in error/reset paths. in gen7_init_oa_buffer()
1442 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1445 static void gen8_init_oa_buffer(struct i915_perf_stream *stream) in gen8_init_oa_buffer() argument
1447 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1448 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1451 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1455 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1472 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1473 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1480 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1482 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1488 * when re-enabling a stream or in error/reset paths. in gen8_init_oa_buffer()
1496 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1499 static void gen12_init_oa_buffer(struct i915_perf_stream *stream) in gen12_init_oa_buffer() argument
1501 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1502 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1505 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1510 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1526 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen12_init_oa_buffer()
1527 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1534 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1536 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1542 * when re-enabling a stream or in error/reset paths. in gen12_init_oa_buffer()
1550 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1551 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1554 static int alloc_oa_buffer(struct i915_perf_stream *stream) in alloc_oa_buffer() argument
1556 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1561 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1567 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1581 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1583 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1585 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1586 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1598 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1599 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1604 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, in save_restore_register() argument
1613 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1619 *cs++ = intel_gt_scratch_offset(stream->engine->gt, in save_restore_register()
1627 static int alloc_noa_wait(struct i915_perf_stream *stream) in alloc_noa_wait() argument
1629 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1633 intel_gt_ns_to_clock_interval(stream->perf->i915->ggtt.vm.gt, in alloc_noa_wait()
1634 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1635 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1665 * needs to be fixed during the lifetime of the i915/perf stream. in alloc_noa_wait()
1682 stream, cs, true /* save */, CS_GPR(i), in alloc_noa_wait()
1685 stream, cs, true /* save */, MI_PREDICATE_RESULT_1, in alloc_noa_wait()
1789 stream, cs, false /* restore */, CS_GPR(i), in alloc_noa_wait()
1792 stream, cs, false /* restore */, MI_PREDICATE_RESULT_1, in alloc_noa_wait()
1803 stream->noa_wait = vma; in alloc_noa_wait()
1854 alloc_oa_config_buffer(struct i915_perf_stream *stream, in alloc_oa_config_buffer() argument
1874 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
1903 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
1906 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
1913 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
1921 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
1942 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config) in get_oa_vma() argument
1948 * to the stream. in get_oa_vma()
1950 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
1958 oa_bo = alloc_oa_config_buffer(stream, oa_config); in get_oa_vma()
1967 emit_oa_config(struct i915_perf_stream *stream, in emit_oa_config() argument
1977 vma = get_oa_vma(stream, oa_config); in emit_oa_config()
2039 static struct intel_context *oa_context(struct i915_perf_stream *stream) in oa_context() argument
2041 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2045 hsw_enable_metric_set(struct i915_perf_stream *stream, in hsw_enable_metric_set() argument
2048 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2065 return emit_oa_config(stream, in hsw_enable_metric_set()
2066 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2070 static void hsw_disable_metric_set(struct i915_perf_stream *stream) in hsw_disable_metric_set() argument
2072 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2112 const struct i915_perf_stream *stream) in gen8_update_reg_state_unlocked() argument
2114 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2115 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2130 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2131 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2136 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2272 static int gen12_configure_oar_context(struct i915_perf_stream *stream, in gen12_configure_oar_context() argument
2276 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2277 u32 format = stream->oa_buffer.format; in gen12_configure_oar_context()
2281 stream->perf->ctx_oactxctrl_offset + 1, in gen12_configure_oar_context()
2321 * Manages updating the per-context aspects of the OA stream
2346 oa_configure_all_contexts(struct i915_perf_stream *stream, in oa_configure_all_contexts() argument
2351 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2356 lockdep_assert_held(&stream->perf->lock); in oa_configure_all_contexts()
2415 gen12_configure_all_contexts(struct i915_perf_stream *stream, in gen12_configure_all_contexts() argument
2426 return oa_configure_all_contexts(stream, in gen12_configure_all_contexts()
2432 lrc_configure_all_contexts(struct i915_perf_stream *stream, in lrc_configure_all_contexts() argument
2437 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2446 stream->perf->ctx_oactxctrl_offset + 1, in lrc_configure_all_contexts()
2460 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2461 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2467 return oa_configure_all_contexts(stream, in lrc_configure_all_contexts()
2473 gen8_enable_metric_set(struct i915_perf_stream *stream, in gen8_enable_metric_set() argument
2476 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2477 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2503 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2514 ret = lrc_configure_all_contexts(stream, oa_config, active); in gen8_enable_metric_set()
2518 return emit_oa_config(stream, in gen8_enable_metric_set()
2519 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2523 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) in oag_report_ctx_switches() argument
2526 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2531 gen12_enable_metric_set(struct i915_perf_stream *stream, in gen12_enable_metric_set() argument
2534 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2535 struct i915_oa_config *oa_config = stream->oa_config; in gen12_enable_metric_set()
2536 bool periodic = stream->periodic; in gen12_enable_metric_set()
2537 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2548 oag_report_ctx_switches(stream)); in gen12_enable_metric_set()
2561 ret = gen12_configure_all_contexts(stream, oa_config, active); in gen12_enable_metric_set()
2570 if (stream->ctx) { in gen12_enable_metric_set()
2571 ret = gen12_configure_oar_context(stream, active); in gen12_enable_metric_set()
2576 return emit_oa_config(stream, in gen12_enable_metric_set()
2577 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2581 static void gen8_disable_metric_set(struct i915_perf_stream *stream) in gen8_disable_metric_set() argument
2583 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2586 lrc_configure_all_contexts(stream, NULL, NULL); in gen8_disable_metric_set()
2591 static void gen11_disable_metric_set(struct i915_perf_stream *stream) in gen11_disable_metric_set() argument
2593 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2596 lrc_configure_all_contexts(stream, NULL, NULL); in gen11_disable_metric_set()
2602 static void gen12_disable_metric_set(struct i915_perf_stream *stream) in gen12_disable_metric_set() argument
2604 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2607 gen12_configure_all_contexts(stream, NULL, NULL); in gen12_disable_metric_set()
2610 if (stream->ctx) in gen12_disable_metric_set()
2611 gen12_configure_oar_context(stream, NULL); in gen12_disable_metric_set()
2617 static void gen7_oa_enable(struct i915_perf_stream *stream) in gen7_oa_enable() argument
2619 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
2620 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2621 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2622 bool periodic = stream->periodic; in gen7_oa_enable()
2623 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2624 u32 report_format = stream->oa_buffer.format; in gen7_oa_enable()
2635 gen7_init_oa_buffer(stream); in gen7_oa_enable()
2647 static void gen8_oa_enable(struct i915_perf_stream *stream) in gen8_oa_enable() argument
2649 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
2650 u32 report_format = stream->oa_buffer.format; in gen8_oa_enable()
2661 gen8_init_oa_buffer(stream); in gen8_oa_enable()
2673 static void gen12_oa_enable(struct i915_perf_stream *stream) in gen12_oa_enable() argument
2675 struct intel_uncore *uncore = stream->uncore; in gen12_oa_enable()
2676 u32 report_format = stream->oa_buffer.format; in gen12_oa_enable()
2682 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
2685 gen12_init_oa_buffer(stream); in gen12_oa_enable()
2693 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
2694 * @stream: An i915 perf stream opened for OA metrics
2697 * when opening the stream. This also starts a hrtimer that will periodically
2701 static void i915_oa_stream_enable(struct i915_perf_stream *stream) in i915_oa_stream_enable() argument
2703 stream->pollin = false; in i915_oa_stream_enable()
2705 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
2707 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
2708 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
2709 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
2713 static void gen7_oa_disable(struct i915_perf_stream *stream) in gen7_oa_disable() argument
2715 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
2721 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
2725 static void gen8_oa_disable(struct i915_perf_stream *stream) in gen8_oa_disable() argument
2727 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
2733 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
2737 static void gen12_oa_disable(struct i915_perf_stream *stream) in gen12_oa_disable() argument
2739 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
2746 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2754 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2759 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
2760 * @stream: An i915 perf stream opened for OA metrics
2766 static void i915_oa_stream_disable(struct i915_perf_stream *stream) in i915_oa_stream_disable() argument
2768 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
2770 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
2771 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
2783 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) in i915_perf_stream_enable_sync() argument
2792 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
2833 * i915_oa_stream_init - validate combined props for OA stream and init
2834 * @stream: An i915 perf stream
2836 * @props: The property state that configures stream (individually validated)
2841 * At this point it has been determined that userspace wants a stream of
2850 static int i915_oa_stream_init(struct i915_perf_stream *stream, in i915_oa_stream_init() argument
2854 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
2855 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
2875 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
2900 stream->engine = props->engine; in i915_oa_stream_init()
2901 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
2903 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
2907 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
2908 stream->sample_size += format_size; in i915_oa_stream_init()
2910 stream->oa_buffer.format_size = format_size; in i915_oa_stream_init()
2911 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0)) in i915_oa_stream_init()
2914 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
2916 stream->oa_buffer.format = in i915_oa_stream_init()
2919 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
2920 if (stream->periodic) in i915_oa_stream_init()
2921 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2923 if (stream->ctx) { in i915_oa_stream_init()
2924 ret = oa_get_render_ctx_id(stream); in i915_oa_stream_init()
2931 ret = alloc_noa_wait(stream); in i915_oa_stream_init()
2937 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
2938 if (!stream->oa_config) { in i915_oa_stream_init()
2956 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
2957 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2959 ret = alloc_oa_buffer(stream); in i915_oa_stream_init()
2963 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
2966 WRITE_ONCE(perf->exclusive_stream, stream); in i915_oa_stream_init()
2968 ret = i915_perf_stream_enable_sync(stream); in i915_oa_stream_init()
2974 DRM_DEBUG("opening stream oa config uuid=%s\n", in i915_oa_stream_init()
2975 stream->oa_config->uuid); in i915_oa_stream_init()
2977 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
2979 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
2980 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
2981 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
2987 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
2989 free_oa_buffer(stream); in i915_oa_stream_init()
2992 free_oa_configs(stream); in i915_oa_stream_init()
2994 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2995 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
2998 free_noa_wait(stream); in i915_oa_stream_init()
3001 if (stream->ctx) in i915_oa_stream_init()
3002 oa_put_render_ctx_id(stream); in i915_oa_stream_init()
3010 struct i915_perf_stream *stream; in i915_oa_init_reg_state() local
3016 stream = READ_ONCE(engine->i915->perf.exclusive_stream); in i915_oa_init_reg_state()
3017 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3018 gen8_update_reg_state_unlocked(ce, stream); in i915_oa_init_reg_state()
3022 * i915_perf_read - handles read() FOP for i915 perf stream FDs
3023 * @file: An i915 perf stream file
3028 * The entry point for handling a read() on a stream file descriptor from
3030 * &i915_perf_stream_ops->read but to save having stream implementations (of
3033 * We can also consistently treat trying to read from a disabled stream
3034 * as an IO error so implementations can assume the stream is enabled
3044 struct i915_perf_stream *stream = file->private_data; in i915_perf_read() local
3045 struct i915_perf *perf = stream->perf; in i915_perf_read()
3050 * disabled stream as an error. In particular it might otherwise lead in i915_perf_read()
3053 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3058 * stream->ops->wait_unlocked. in i915_perf_read()
3065 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3070 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3075 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3091 stream->pollin = false; in i915_perf_read()
3099 struct i915_perf_stream *stream = in oa_poll_check_timer_cb() local
3100 container_of(hrtimer, typeof(*stream), poll_check_timer); in oa_poll_check_timer_cb()
3102 if (oa_buffer_check_unlocked(stream)) { in oa_poll_check_timer_cb()
3103 stream->pollin = true; in oa_poll_check_timer_cb()
3104 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3108 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3114 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
3115 * @stream: An i915 perf stream
3116 * @file: An i915 perf stream file
3119 * For handling userspace polling on an i915 perf stream, this calls through to
3121 * will be woken for new stream data.
3128 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream, in i915_perf_poll_locked() argument
3134 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3142 if (stream->pollin) in i915_perf_poll_locked()
3149 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
3150 * @file: An i915 perf stream file
3153 * For handling userspace polling on an i915 perf stream, this ensures
3154 * poll_wait() gets called with a wait queue that will be woken for new stream
3163 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll() local
3164 struct i915_perf *perf = stream->perf; in i915_perf_poll()
3168 ret = i915_perf_poll_locked(stream, file, wait); in i915_perf_poll()
3176 * @stream: A disabled i915 perf stream
3178 * [Re]enables the associated capture of data for this stream.
3180 * If a stream was previously enabled then there's currently no intention
3184 static void i915_perf_enable_locked(struct i915_perf_stream *stream) in i915_perf_enable_locked() argument
3186 if (stream->enabled) in i915_perf_enable_locked()
3189 /* Allow stream->ops->enable() to refer to this */ in i915_perf_enable_locked()
3190 stream->enabled = true; in i915_perf_enable_locked()
3192 if (stream->ops->enable) in i915_perf_enable_locked()
3193 stream->ops->enable(stream); in i915_perf_enable_locked()
3195 if (stream->hold_preemption) in i915_perf_enable_locked()
3196 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3201 * @stream: An enabled i915 perf stream
3203 * Disables the associated capture of data for this stream.
3205 * The intention is that disabling an re-enabling a stream will ideally be
3206 * cheaper than destroying and re-opening a stream with the same configuration,
3208 * must be retained between disabling and re-enabling a stream.
3210 * Note: while a stream is disabled it's considered an error for userspace
3211 * to attempt to read from the stream (-EIO).
3213 static void i915_perf_disable_locked(struct i915_perf_stream *stream) in i915_perf_disable_locked() argument
3215 if (!stream->enabled) in i915_perf_disable_locked()
3218 /* Allow stream->ops->disable() to refer to this */ in i915_perf_disable_locked()
3219 stream->enabled = false; in i915_perf_disable_locked()
3221 if (stream->hold_preemption) in i915_perf_disable_locked()
3222 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3224 if (stream->ops->disable) in i915_perf_disable_locked()
3225 stream->ops->disable(stream); in i915_perf_disable_locked()
3228 static long i915_perf_config_locked(struct i915_perf_stream *stream, in i915_perf_config_locked() argument
3232 long ret = stream->oa_config->id; in i915_perf_config_locked()
3234 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3238 if (config != stream->oa_config) { in i915_perf_config_locked()
3250 err = emit_oa_config(stream, config, oa_context(stream), NULL); in i915_perf_config_locked()
3252 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3263 * i915_perf_ioctl_locked - support ioctl() usage with i915 perf stream FDs
3264 * @stream: An i915 perf stream
3274 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, in i915_perf_ioctl_locked() argument
3280 i915_perf_enable_locked(stream); in i915_perf_ioctl_locked()
3283 i915_perf_disable_locked(stream); in i915_perf_ioctl_locked()
3286 return i915_perf_config_locked(stream, arg); in i915_perf_ioctl_locked()
3293 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
3294 * @file: An i915 perf stream file
3307 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl() local
3308 struct i915_perf *perf = stream->perf; in i915_perf_ioctl()
3312 ret = i915_perf_ioctl_locked(stream, cmd, arg); in i915_perf_ioctl()
3319 * i915_perf_destroy_locked - destroy an i915 perf stream
3320 * @stream: An i915 perf stream
3322 * Frees all resources associated with the given i915 perf @stream, disabling
3328 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) in i915_perf_destroy_locked() argument
3330 if (stream->enabled) in i915_perf_destroy_locked()
3331 i915_perf_disable_locked(stream); in i915_perf_destroy_locked()
3333 if (stream->ops->destroy) in i915_perf_destroy_locked()
3334 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3336 if (stream->ctx) in i915_perf_destroy_locked()
3337 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3339 kfree(stream); in i915_perf_destroy_locked()
3343 * i915_perf_release - handles userspace close() of a stream file
3345 * @file: An i915 perf stream file
3347 * Cleans up any resources associated with an open i915 perf stream file.
3355 struct i915_perf_stream *stream = file->private_data; in i915_perf_release() local
3356 struct i915_perf *perf = stream->perf; in i915_perf_release()
3359 i915_perf_destroy_locked(stream); in i915_perf_release()
3362 /* Release the reference the perf stream kept on the driver. */ in i915_perf_release()
3384 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
3392 * Implements further stream config validation and stream initialization on
3401 * config validation and stream initialization details will be handled by
3403 * will be relevant to all stream types / backends.
3414 struct i915_perf_stream *stream = NULL; in i915_perf_open_ioctl_locked() local
3426 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n", in i915_perf_open_ioctl_locked()
3449 * doesn't request global stream access (i.e. query based sampling in i915_perf_open_ioctl_locked()
3482 DRM_DEBUG("Insufficient privileges to open i915 perf stream\n"); in i915_perf_open_ioctl_locked()
3487 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in i915_perf_open_ioctl_locked()
3488 if (!stream) { in i915_perf_open_ioctl_locked()
3493 stream->perf = perf; in i915_perf_open_ioctl_locked()
3494 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3495 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3497 ret = i915_oa_stream_init(stream, param, props); in i915_perf_open_ioctl_locked()
3501 /* we avoid simply assigning stream->sample_flags = props->sample_flags in i915_perf_open_ioctl_locked()
3505 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3515 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); in i915_perf_open_ioctl_locked()
3522 i915_perf_enable_locked(stream); in i915_perf_open_ioctl_locked()
3532 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3533 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3535 kfree(stream); in i915_perf_open_ioctl_locked()
3562 * read_properties_unlocked - validate + copy userspace stream open properties
3566 * @props: The stream configuration built up while validating properties
3570 * properties necessary for a particular kind of stream have been set.
3738 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
3743 * Validates the stream open parameters given by userspace including flags
3746 * Very little is assumed up front about the nature of the stream being
3748 * i915-perf stream is expected to be a suitable interface for other forms of
3758 * Return: A newly opened i915 Perf stream file descriptor or negative
3803 * used to open an i915-perf stream.
4202 * and their content will be freed when the stream using the config is closed.
4456 * stream instead of waiting until driver _fini which no one in i915_perf_init()