Lines Matching refs:stream
415 static u32 gen12_oa_hw_tail_read(struct i915_perf_stream *stream) in gen12_oa_hw_tail_read() argument
417 struct intel_uncore *uncore = stream->uncore; in gen12_oa_hw_tail_read()
423 static u32 gen8_oa_hw_tail_read(struct i915_perf_stream *stream) in gen8_oa_hw_tail_read() argument
425 struct intel_uncore *uncore = stream->uncore; in gen8_oa_hw_tail_read()
430 static u32 gen7_oa_hw_tail_read(struct i915_perf_stream *stream) in gen7_oa_hw_tail_read() argument
432 struct intel_uncore *uncore = stream->uncore; in gen7_oa_hw_tail_read()
462 static bool oa_buffer_check_unlocked(struct i915_perf_stream *stream) in oa_buffer_check_unlocked() argument
464 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in oa_buffer_check_unlocked()
465 int report_size = stream->oa_buffer.format_size; in oa_buffer_check_unlocked()
475 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
477 hw_tail = stream->perf->ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
486 if (hw_tail == stream->oa_buffer.aging_tail && in oa_buffer_check_unlocked()
487 (now - stream->oa_buffer.aging_timestamp) > OA_TAIL_MARGIN_NSEC) { in oa_buffer_check_unlocked()
492 stream->oa_buffer.tail = stream->oa_buffer.aging_tail; in oa_buffer_check_unlocked()
500 head = stream->oa_buffer.head - gtt_offset; in oa_buffer_check_unlocked()
501 aged_tail = stream->oa_buffer.tail - gtt_offset; in oa_buffer_check_unlocked()
518 u32 *report32 = (void *)(stream->oa_buffer.vaddr + tail); in oa_buffer_check_unlocked()
527 __ratelimit(&stream->perf->tail_pointer_race)) in oa_buffer_check_unlocked()
532 stream->oa_buffer.tail = gtt_offset + tail; in oa_buffer_check_unlocked()
533 stream->oa_buffer.aging_tail = gtt_offset + hw_tail; in oa_buffer_check_unlocked()
534 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
537 pollin = OA_TAKEN(stream->oa_buffer.tail - gtt_offset, in oa_buffer_check_unlocked()
538 stream->oa_buffer.head - gtt_offset) >= report_size; in oa_buffer_check_unlocked()
540 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
560 static int append_oa_status(struct i915_perf_stream *stream, in append_oa_status() argument
596 static int append_oa_sample(struct i915_perf_stream *stream, in append_oa_sample() argument
602 int report_size = stream->oa_buffer.format_size; in append_oa_sample()
607 header.size = stream->sample_size; in append_oa_sample()
646 static int gen8_append_oa_reports(struct i915_perf_stream *stream, in gen8_append_oa_reports() argument
651 struct intel_uncore *uncore = stream->uncore; in gen8_append_oa_reports()
652 int report_size = stream->oa_buffer.format_size; in gen8_append_oa_reports()
653 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
654 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
662 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen8_append_oa_reports()
665 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
667 head = stream->oa_buffer.head; in gen8_append_oa_reports()
668 tail = stream->oa_buffer.tail; in gen8_append_oa_reports()
670 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
728 (GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
732 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
742 if (!(report32[0] & stream->perf->gen8_valid_ctx_bit) && in gen8_append_oa_reports()
743 GRAPHICS_VER(stream->perf->i915) <= 11) in gen8_append_oa_reports()
777 if (!stream->perf->exclusive_stream->ctx || in gen8_append_oa_reports()
778 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
779 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
786 if (stream->perf->exclusive_stream->ctx && in gen8_append_oa_reports()
787 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
791 ret = append_oa_sample(stream, buf, count, offset, in gen8_append_oa_reports()
796 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
810 oaheadptr = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_append_oa_reports()
813 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
822 stream->oa_buffer.head = head; in gen8_append_oa_reports()
824 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
850 static int gen8_oa_read(struct i915_perf_stream *stream, in gen8_oa_read() argument
855 struct intel_uncore *uncore = stream->uncore; in gen8_oa_read()
860 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen8_oa_read()
863 oastatus_reg = GRAPHICS_VER(stream->perf->i915) == 12 ? in gen8_oa_read()
883 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
888 drm_dbg(&stream->perf->i915->drm, in gen8_oa_read()
890 stream->period_exponent); in gen8_oa_read()
892 stream->perf->ops.oa_disable(stream); in gen8_oa_read()
893 stream->perf->ops.oa_enable(stream); in gen8_oa_read()
903 ret = append_oa_status(stream, buf, count, offset, in gen8_oa_read()
916 return gen8_append_oa_reports(stream, buf, count, offset); in gen8_oa_read()
940 static int gen7_append_oa_reports(struct i915_perf_stream *stream, in gen7_append_oa_reports() argument
945 struct intel_uncore *uncore = stream->uncore; in gen7_append_oa_reports()
946 int report_size = stream->oa_buffer.format_size; in gen7_append_oa_reports()
947 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
948 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
956 if (drm_WARN_ON(&uncore->i915->drm, !stream->enabled)) in gen7_append_oa_reports()
959 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
961 head = stream->oa_buffer.head; in gen7_append_oa_reports()
962 tail = stream->oa_buffer.tail; in gen7_append_oa_reports()
964 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1014 if (__ratelimit(&stream->perf->spurious_report_rs)) in gen7_append_oa_reports()
1019 ret = append_oa_sample(stream, buf, count, offset, report); in gen7_append_oa_reports()
1031 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1041 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1043 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1065 static int gen7_oa_read(struct i915_perf_stream *stream, in gen7_oa_read() argument
1070 struct intel_uncore *uncore = stream->uncore; in gen7_oa_read()
1074 if (drm_WARN_ON(&uncore->i915->drm, !stream->oa_buffer.vaddr)) in gen7_oa_read()
1084 oastatus1 &= ~stream->perf->gen7_latched_oastatus1; in gen7_oa_read()
1107 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1112 drm_dbg(&stream->perf->i915->drm, in gen7_oa_read()
1114 stream->period_exponent); in gen7_oa_read()
1116 stream->perf->ops.oa_disable(stream); in gen7_oa_read()
1117 stream->perf->ops.oa_enable(stream); in gen7_oa_read()
1123 ret = append_oa_status(stream, buf, count, offset, in gen7_oa_read()
1127 stream->perf->gen7_latched_oastatus1 |= in gen7_oa_read()
1131 return gen7_append_oa_reports(stream, buf, count, offset); in gen7_oa_read()
1148 static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) in i915_oa_wait_unlocked() argument
1151 if (!stream->periodic) in i915_oa_wait_unlocked()
1154 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1155 oa_buffer_check_unlocked(stream)); in i915_oa_wait_unlocked()
1168 static void i915_oa_poll_wait(struct i915_perf_stream *stream, in i915_oa_poll_wait() argument
1172 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1187 static int i915_oa_read(struct i915_perf_stream *stream, in i915_oa_read() argument
1192 return stream->perf->ops.read(stream, buf, count, offset); in i915_oa_read()
1195 static struct intel_context *oa_pin_context(struct i915_perf_stream *stream) in oa_pin_context() argument
1198 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1204 if (ce->engine != stream->engine) /* first match! */ in oa_pin_context()
1232 stream->pinned_ctx = ce; in oa_pin_context()
1233 return stream->pinned_ctx; in oa_pin_context()
1246 static int oa_get_render_ctx_id(struct i915_perf_stream *stream) in oa_get_render_ctx_id() argument
1250 ce = oa_pin_context(stream); in oa_get_render_ctx_id()
1260 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1261 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1278 stream->specific_ctx_id = ce->lrc.lrca >> 12; in oa_get_render_ctx_id()
1284 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1287 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1289 stream->specific_ctx_id = stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1296 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1299 stream->specific_ctx_id = in oa_get_render_ctx_id()
1303 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1310 stream->specific_ctx_id = in oa_get_render_ctx_id()
1319 ce->tag = stream->specific_ctx_id; in oa_get_render_ctx_id()
1321 drm_dbg(&stream->perf->i915->drm, in oa_get_render_ctx_id()
1323 stream->specific_ctx_id, in oa_get_render_ctx_id()
1324 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1336 static void oa_put_render_ctx_id(struct i915_perf_stream *stream) in oa_put_render_ctx_id() argument
1340 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1346 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1347 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1351 free_oa_buffer(struct i915_perf_stream *stream) in free_oa_buffer() argument
1353 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1356 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1360 free_oa_configs(struct i915_perf_stream *stream) in free_oa_configs() argument
1364 i915_oa_config_put(stream->oa_config); in free_oa_configs()
1365 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in free_oa_configs()
1370 free_noa_wait(struct i915_perf_stream *stream) in free_noa_wait() argument
1372 i915_vma_unpin_and_release(&stream->noa_wait, 0); in free_noa_wait()
1375 static void i915_oa_stream_destroy(struct i915_perf_stream *stream) in i915_oa_stream_destroy() argument
1377 struct i915_perf *perf = stream->perf; in i915_oa_stream_destroy()
1379 if (WARN_ON(stream != perf->exclusive_stream)) in i915_oa_stream_destroy()
1389 perf->ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1391 free_oa_buffer(stream); in i915_oa_stream_destroy()
1393 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1394 intel_engine_pm_put(stream->engine); in i915_oa_stream_destroy()
1396 if (stream->ctx) in i915_oa_stream_destroy()
1397 oa_put_render_ctx_id(stream); in i915_oa_stream_destroy()
1399 free_oa_configs(stream); in i915_oa_stream_destroy()
1400 free_noa_wait(stream); in i915_oa_stream_destroy()
1408 static void gen7_init_oa_buffer(struct i915_perf_stream *stream) in gen7_init_oa_buffer() argument
1410 struct intel_uncore *uncore = stream->uncore; in gen7_init_oa_buffer()
1411 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1414 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1421 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1429 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1430 stream->oa_buffer.tail = gtt_offset; in gen7_init_oa_buffer()
1432 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1438 stream->perf->gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1451 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1454 static void gen8_init_oa_buffer(struct i915_perf_stream *stream) in gen8_init_oa_buffer() argument
1456 struct intel_uncore *uncore = stream->uncore; in gen8_init_oa_buffer()
1457 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1460 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1464 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1481 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1482 stream->oa_buffer.tail = gtt_offset; in gen8_init_oa_buffer()
1489 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1491 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1505 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1508 static void gen12_init_oa_buffer(struct i915_perf_stream *stream) in gen12_init_oa_buffer() argument
1510 struct intel_uncore *uncore = stream->uncore; in gen12_init_oa_buffer()
1511 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen12_init_oa_buffer()
1514 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1519 stream->oa_buffer.head = gtt_offset; in gen12_init_oa_buffer()
1535 stream->oa_buffer.aging_tail = INVALID_TAIL_PTR; in gen12_init_oa_buffer()
1536 stream->oa_buffer.tail = gtt_offset; in gen12_init_oa_buffer()
1543 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen12_init_oa_buffer()
1545 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen12_init_oa_buffer()
1559 memset(stream->oa_buffer.vaddr, 0, in gen12_init_oa_buffer()
1560 stream->oa_buffer.vma->size); in gen12_init_oa_buffer()
1563 static int alloc_oa_buffer(struct i915_perf_stream *stream) in alloc_oa_buffer() argument
1565 struct drm_i915_private *i915 = stream->perf->i915; in alloc_oa_buffer()
1570 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.vma)) in alloc_oa_buffer()
1576 bo = i915_gem_object_create_shmem(stream->perf->i915, OA_BUFFER_SIZE); in alloc_oa_buffer()
1590 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1592 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1594 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1595 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1607 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1608 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1613 static u32 *save_restore_register(struct i915_perf_stream *stream, u32 *cs, in save_restore_register() argument
1622 if (GRAPHICS_VER(stream->perf->i915) >= 8) in save_restore_register()
1628 *cs++ = intel_gt_scratch_offset(stream->engine->gt, in save_restore_register()
1636 static int alloc_noa_wait(struct i915_perf_stream *stream) in alloc_noa_wait() argument
1638 struct drm_i915_private *i915 = stream->perf->i915; in alloc_noa_wait()
1642 intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), in alloc_noa_wait()
1643 atomic64_read(&stream->perf->noa_programming_delay)); in alloc_noa_wait()
1644 const u32 base = stream->engine->mmio_base; in alloc_noa_wait()
1691 stream, cs, true /* save */, CS_GPR(i), in alloc_noa_wait()
1694 stream, cs, true /* save */, MI_PREDICATE_RESULT_1(RENDER_RING_BASE), in alloc_noa_wait()
1798 stream, cs, false /* restore */, CS_GPR(i), in alloc_noa_wait()
1801 stream, cs, false /* restore */, MI_PREDICATE_RESULT_1(RENDER_RING_BASE), in alloc_noa_wait()
1812 stream->noa_wait = vma; in alloc_noa_wait()
1863 alloc_oa_config_buffer(struct i915_perf_stream *stream, in alloc_oa_config_buffer() argument
1883 obj = i915_gem_object_create_shmem(stream->perf->i915, config_length); in alloc_oa_config_buffer()
1912 *cs++ = (GRAPHICS_VER(stream->perf->i915) < 8 ? in alloc_oa_config_buffer()
1915 *cs++ = i915_ggtt_offset(stream->noa_wait); in alloc_oa_config_buffer()
1922 &stream->engine->gt->ggtt->vm, in alloc_oa_config_buffer()
1930 llist_add(&oa_bo->node, &stream->oa_config_bos); in alloc_oa_config_buffer()
1951 get_oa_vma(struct i915_perf_stream *stream, struct i915_oa_config *oa_config) in get_oa_vma() argument
1959 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in get_oa_vma()
1967 oa_bo = alloc_oa_config_buffer(stream, oa_config); in get_oa_vma()
1976 emit_oa_config(struct i915_perf_stream *stream, in emit_oa_config() argument
1986 vma = get_oa_vma(stream, oa_config); in emit_oa_config()
2048 static struct intel_context *oa_context(struct i915_perf_stream *stream) in oa_context() argument
2050 return stream->pinned_ctx ?: stream->engine->kernel_context; in oa_context()
2054 hsw_enable_metric_set(struct i915_perf_stream *stream, in hsw_enable_metric_set() argument
2057 struct intel_uncore *uncore = stream->uncore; in hsw_enable_metric_set()
2074 return emit_oa_config(stream, in hsw_enable_metric_set()
2075 stream->oa_config, oa_context(stream), in hsw_enable_metric_set()
2079 static void hsw_disable_metric_set(struct i915_perf_stream *stream) in hsw_disable_metric_set() argument
2081 struct intel_uncore *uncore = stream->uncore; in hsw_disable_metric_set()
2121 const struct i915_perf_stream *stream) in gen8_update_reg_state_unlocked() argument
2123 u32 ctx_oactxctrl = stream->perf->ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
2124 u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
2139 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
2140 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
2145 oa_config_flex_reg(stream->oa_config, flex_regs[i]); in gen8_update_reg_state_unlocked()
2281 static int gen12_configure_oar_context(struct i915_perf_stream *stream, in gen12_configure_oar_context() argument
2285 struct intel_context *ce = stream->pinned_ctx; in gen12_configure_oar_context()
2286 u32 format = stream->oa_buffer.format; in gen12_configure_oar_context()
2290 stream->perf->ctx_oactxctrl_offset + 1, in gen12_configure_oar_context()
2355 oa_configure_all_contexts(struct i915_perf_stream *stream, in oa_configure_all_contexts() argument
2360 struct drm_i915_private *i915 = stream->perf->i915; in oa_configure_all_contexts()
2365 lockdep_assert_held(&stream->perf->lock); in oa_configure_all_contexts()
2424 gen12_configure_all_contexts(struct i915_perf_stream *stream, in gen12_configure_all_contexts() argument
2435 return oa_configure_all_contexts(stream, in gen12_configure_all_contexts()
2441 lrc_configure_all_contexts(struct i915_perf_stream *stream, in lrc_configure_all_contexts() argument
2446 const u32 ctx_flexeu0 = stream->perf->ctx_flexeu0_offset; in lrc_configure_all_contexts()
2455 stream->perf->ctx_oactxctrl_offset + 1, in lrc_configure_all_contexts()
2469 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in lrc_configure_all_contexts()
2470 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in lrc_configure_all_contexts()
2476 return oa_configure_all_contexts(stream, in lrc_configure_all_contexts()
2482 gen8_enable_metric_set(struct i915_perf_stream *stream, in gen8_enable_metric_set() argument
2485 struct intel_uncore *uncore = stream->uncore; in gen8_enable_metric_set()
2486 struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
2512 if (IS_GRAPHICS_VER(stream->perf->i915, 9, 11)) { in gen8_enable_metric_set()
2523 ret = lrc_configure_all_contexts(stream, oa_config, active); in gen8_enable_metric_set()
2527 return emit_oa_config(stream, in gen8_enable_metric_set()
2528 stream->oa_config, oa_context(stream), in gen8_enable_metric_set()
2532 static u32 oag_report_ctx_switches(const struct i915_perf_stream *stream) in oag_report_ctx_switches() argument
2535 (stream->sample_flags & SAMPLE_OA_REPORT) ? in oag_report_ctx_switches()
2540 gen12_enable_metric_set(struct i915_perf_stream *stream, in gen12_enable_metric_set() argument
2543 struct intel_uncore *uncore = stream->uncore; in gen12_enable_metric_set()
2544 struct i915_oa_config *oa_config = stream->oa_config; in gen12_enable_metric_set()
2545 bool periodic = stream->periodic; in gen12_enable_metric_set()
2546 u32 period_exponent = stream->period_exponent; in gen12_enable_metric_set()
2557 oag_report_ctx_switches(stream)); in gen12_enable_metric_set()
2570 ret = gen12_configure_all_contexts(stream, oa_config, active); in gen12_enable_metric_set()
2579 if (stream->ctx) { in gen12_enable_metric_set()
2580 ret = gen12_configure_oar_context(stream, active); in gen12_enable_metric_set()
2585 return emit_oa_config(stream, in gen12_enable_metric_set()
2586 stream->oa_config, oa_context(stream), in gen12_enable_metric_set()
2590 static void gen8_disable_metric_set(struct i915_perf_stream *stream) in gen8_disable_metric_set() argument
2592 struct intel_uncore *uncore = stream->uncore; in gen8_disable_metric_set()
2595 lrc_configure_all_contexts(stream, NULL, NULL); in gen8_disable_metric_set()
2600 static void gen11_disable_metric_set(struct i915_perf_stream *stream) in gen11_disable_metric_set() argument
2602 struct intel_uncore *uncore = stream->uncore; in gen11_disable_metric_set()
2605 lrc_configure_all_contexts(stream, NULL, NULL); in gen11_disable_metric_set()
2611 static void gen12_disable_metric_set(struct i915_perf_stream *stream) in gen12_disable_metric_set() argument
2613 struct intel_uncore *uncore = stream->uncore; in gen12_disable_metric_set()
2616 gen12_configure_all_contexts(stream, NULL, NULL); in gen12_disable_metric_set()
2619 if (stream->ctx) in gen12_disable_metric_set()
2620 gen12_configure_oar_context(stream, NULL); in gen12_disable_metric_set()
2626 static void gen7_oa_enable(struct i915_perf_stream *stream) in gen7_oa_enable() argument
2628 struct intel_uncore *uncore = stream->uncore; in gen7_oa_enable()
2629 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2630 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2631 bool periodic = stream->periodic; in gen7_oa_enable()
2632 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2633 u32 report_format = stream->oa_buffer.format; in gen7_oa_enable()
2644 gen7_init_oa_buffer(stream); in gen7_oa_enable()
2656 static void gen8_oa_enable(struct i915_perf_stream *stream) in gen8_oa_enable() argument
2658 struct intel_uncore *uncore = stream->uncore; in gen8_oa_enable()
2659 u32 report_format = stream->oa_buffer.format; in gen8_oa_enable()
2670 gen8_init_oa_buffer(stream); in gen8_oa_enable()
2682 static void gen12_oa_enable(struct i915_perf_stream *stream) in gen12_oa_enable() argument
2684 struct intel_uncore *uncore = stream->uncore; in gen12_oa_enable()
2685 u32 report_format = stream->oa_buffer.format; in gen12_oa_enable()
2691 if (!(stream->sample_flags & SAMPLE_OA_REPORT)) in gen12_oa_enable()
2694 gen12_init_oa_buffer(stream); in gen12_oa_enable()
2710 static void i915_oa_stream_enable(struct i915_perf_stream *stream) in i915_oa_stream_enable() argument
2712 stream->pollin = false; in i915_oa_stream_enable()
2714 stream->perf->ops.oa_enable(stream); in i915_oa_stream_enable()
2716 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_enable()
2717 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
2718 ns_to_ktime(stream->poll_oa_period), in i915_oa_stream_enable()
2722 static void gen7_oa_disable(struct i915_perf_stream *stream) in gen7_oa_disable() argument
2724 struct intel_uncore *uncore = stream->uncore; in gen7_oa_disable()
2730 drm_err(&stream->perf->i915->drm, in gen7_oa_disable()
2734 static void gen8_oa_disable(struct i915_perf_stream *stream) in gen8_oa_disable() argument
2736 struct intel_uncore *uncore = stream->uncore; in gen8_oa_disable()
2742 drm_err(&stream->perf->i915->drm, in gen8_oa_disable()
2746 static void gen12_oa_disable(struct i915_perf_stream *stream) in gen12_oa_disable() argument
2748 struct intel_uncore *uncore = stream->uncore; in gen12_oa_disable()
2755 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2763 drm_err(&stream->perf->i915->drm, in gen12_oa_disable()
2775 static void i915_oa_stream_disable(struct i915_perf_stream *stream) in i915_oa_stream_disable() argument
2777 stream->perf->ops.oa_disable(stream); in i915_oa_stream_disable()
2779 if (stream->sample_flags & SAMPLE_OA_REPORT) in i915_oa_stream_disable()
2780 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
2792 static int i915_perf_stream_enable_sync(struct i915_perf_stream *stream) in i915_perf_stream_enable_sync() argument
2801 err = stream->perf->ops.enable_metric_set(stream, active); in i915_perf_stream_enable_sync()
2859 static int i915_oa_stream_init(struct i915_perf_stream *stream, in i915_oa_stream_init() argument
2863 struct drm_i915_private *i915 = stream->perf->i915; in i915_oa_stream_init()
2864 struct i915_perf *perf = stream->perf; in i915_oa_stream_init()
2869 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2880 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2886 (GRAPHICS_VER(perf->i915) < 12 || !stream->ctx)) { in i915_oa_stream_init()
2887 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2893 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2904 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2910 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2915 stream->engine = props->engine; in i915_oa_stream_init()
2916 stream->uncore = stream->engine->gt->uncore; in i915_oa_stream_init()
2918 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
2922 stream->sample_flags = props->sample_flags; in i915_oa_stream_init()
2923 stream->sample_size += format_size; in i915_oa_stream_init()
2925 stream->oa_buffer.format_size = format_size; in i915_oa_stream_init()
2926 if (drm_WARN_ON(&i915->drm, stream->oa_buffer.format_size == 0)) in i915_oa_stream_init()
2929 stream->hold_preemption = props->hold_preemption; in i915_oa_stream_init()
2931 stream->oa_buffer.format = in i915_oa_stream_init()
2934 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
2935 if (stream->periodic) in i915_oa_stream_init()
2936 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2938 if (stream->ctx) { in i915_oa_stream_init()
2939 ret = oa_get_render_ctx_id(stream); in i915_oa_stream_init()
2941 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2947 ret = alloc_noa_wait(stream); in i915_oa_stream_init()
2949 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2954 stream->oa_config = i915_perf_get_oa_config(perf, props->metrics_set); in i915_oa_stream_init()
2955 if (!stream->oa_config) { in i915_oa_stream_init()
2956 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2974 intel_engine_pm_get(stream->engine); in i915_oa_stream_init()
2975 intel_uncore_forcewake_get(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2977 ret = alloc_oa_buffer(stream); in i915_oa_stream_init()
2981 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
2984 WRITE_ONCE(perf->exclusive_stream, stream); in i915_oa_stream_init()
2986 ret = i915_perf_stream_enable_sync(stream); in i915_oa_stream_init()
2988 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2993 drm_dbg(&stream->perf->i915->drm, in i915_oa_stream_init()
2995 stream->oa_config->uuid); in i915_oa_stream_init()
2997 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
2999 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
3000 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
3001 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
3007 perf->ops.disable_metric_set(stream); in i915_oa_stream_init()
3009 free_oa_buffer(stream); in i915_oa_stream_init()
3012 free_oa_configs(stream); in i915_oa_stream_init()
3014 intel_uncore_forcewake_put(stream->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
3015 intel_engine_pm_put(stream->engine); in i915_oa_stream_init()
3018 free_noa_wait(stream); in i915_oa_stream_init()
3021 if (stream->ctx) in i915_oa_stream_init()
3022 oa_put_render_ctx_id(stream); in i915_oa_stream_init()
3030 struct i915_perf_stream *stream; in i915_oa_init_reg_state() local
3036 stream = READ_ONCE(engine->i915->perf.exclusive_stream); in i915_oa_init_reg_state()
3037 if (stream && GRAPHICS_VER(stream->perf->i915) < 12) in i915_oa_init_reg_state()
3038 gen8_update_reg_state_unlocked(ce, stream); in i915_oa_init_reg_state()
3064 struct i915_perf_stream *stream = file->private_data; in i915_perf_read() local
3065 struct i915_perf *perf = stream->perf; in i915_perf_read()
3073 if (!stream->enabled || !(stream->sample_flags & SAMPLE_OA_REPORT)) in i915_perf_read()
3085 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
3090 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3095 ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read()
3111 stream->pollin = false; in i915_perf_read()
3119 struct i915_perf_stream *stream = in oa_poll_check_timer_cb() local
3120 container_of(hrtimer, typeof(*stream), poll_check_timer); in oa_poll_check_timer_cb()
3122 if (oa_buffer_check_unlocked(stream)) { in oa_poll_check_timer_cb()
3123 stream->pollin = true; in oa_poll_check_timer_cb()
3124 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
3128 ns_to_ktime(stream->poll_oa_period)); in oa_poll_check_timer_cb()
3148 static __poll_t i915_perf_poll_locked(struct i915_perf_stream *stream, in i915_perf_poll_locked() argument
3154 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
3162 if (stream->pollin) in i915_perf_poll_locked()
3183 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll() local
3184 struct i915_perf *perf = stream->perf; in i915_perf_poll()
3188 ret = i915_perf_poll_locked(stream, file, wait); in i915_perf_poll()
3204 static void i915_perf_enable_locked(struct i915_perf_stream *stream) in i915_perf_enable_locked() argument
3206 if (stream->enabled) in i915_perf_enable_locked()
3210 stream->enabled = true; in i915_perf_enable_locked()
3212 if (stream->ops->enable) in i915_perf_enable_locked()
3213 stream->ops->enable(stream); in i915_perf_enable_locked()
3215 if (stream->hold_preemption) in i915_perf_enable_locked()
3216 intel_context_set_nopreempt(stream->pinned_ctx); in i915_perf_enable_locked()
3233 static void i915_perf_disable_locked(struct i915_perf_stream *stream) in i915_perf_disable_locked() argument
3235 if (!stream->enabled) in i915_perf_disable_locked()
3239 stream->enabled = false; in i915_perf_disable_locked()
3241 if (stream->hold_preemption) in i915_perf_disable_locked()
3242 intel_context_clear_nopreempt(stream->pinned_ctx); in i915_perf_disable_locked()
3244 if (stream->ops->disable) in i915_perf_disable_locked()
3245 stream->ops->disable(stream); in i915_perf_disable_locked()
3248 static long i915_perf_config_locked(struct i915_perf_stream *stream, in i915_perf_config_locked() argument
3252 long ret = stream->oa_config->id; in i915_perf_config_locked()
3254 config = i915_perf_get_oa_config(stream->perf, metrics_set); in i915_perf_config_locked()
3258 if (config != stream->oa_config) { in i915_perf_config_locked()
3270 err = emit_oa_config(stream, config, oa_context(stream), NULL); in i915_perf_config_locked()
3272 config = xchg(&stream->oa_config, config); in i915_perf_config_locked()
3294 static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, in i915_perf_ioctl_locked() argument
3300 i915_perf_enable_locked(stream); in i915_perf_ioctl_locked()
3303 i915_perf_disable_locked(stream); in i915_perf_ioctl_locked()
3306 return i915_perf_config_locked(stream, arg); in i915_perf_ioctl_locked()
3327 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl() local
3328 struct i915_perf *perf = stream->perf; in i915_perf_ioctl()
3332 ret = i915_perf_ioctl_locked(stream, cmd, arg); in i915_perf_ioctl()
3348 static void i915_perf_destroy_locked(struct i915_perf_stream *stream) in i915_perf_destroy_locked() argument
3350 if (stream->enabled) in i915_perf_destroy_locked()
3351 i915_perf_disable_locked(stream); in i915_perf_destroy_locked()
3353 if (stream->ops->destroy) in i915_perf_destroy_locked()
3354 stream->ops->destroy(stream); in i915_perf_destroy_locked()
3356 if (stream->ctx) in i915_perf_destroy_locked()
3357 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
3359 kfree(stream); in i915_perf_destroy_locked()
3375 struct i915_perf_stream *stream = file->private_data; in i915_perf_release() local
3376 struct i915_perf *perf = stream->perf; in i915_perf_release()
3379 i915_perf_destroy_locked(stream); in i915_perf_release()
3434 struct i915_perf_stream *stream = NULL; in i915_perf_open_ioctl_locked() local
3510 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in i915_perf_open_ioctl_locked()
3511 if (!stream) { in i915_perf_open_ioctl_locked()
3516 stream->perf = perf; in i915_perf_open_ioctl_locked()
3517 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
3518 stream->poll_oa_period = props->poll_oa_period; in i915_perf_open_ioctl_locked()
3520 ret = i915_oa_stream_init(stream, param, props); in i915_perf_open_ioctl_locked()
3528 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
3538 stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); in i915_perf_open_ioctl_locked()
3545 i915_perf_enable_locked(stream); in i915_perf_open_ioctl_locked()
3555 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
3556 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
3558 kfree(stream); in i915_perf_open_ioctl_locked()