Lines Matching refs:dev_priv
365 static void free_oa_config(struct drm_i915_private *dev_priv, in free_oa_config() argument
377 static void put_oa_config(struct drm_i915_private *dev_priv, in put_oa_config() argument
383 free_oa_config(dev_priv, oa_config); in put_oa_config()
386 static int get_oa_config(struct drm_i915_private *dev_priv, in get_oa_config() argument
393 *out_config = &dev_priv->perf.oa.test_config; in get_oa_config()
394 atomic_inc(&dev_priv->perf.oa.test_config.ref_count); in get_oa_config()
398 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in get_oa_config()
402 *out_config = idr_find(&dev_priv->perf.metrics_idr, metrics_set); in get_oa_config()
408 mutex_unlock(&dev_priv->perf.metrics_lock); in get_oa_config()
413 static u32 gen8_oa_hw_tail_read(struct drm_i915_private *dev_priv) in gen8_oa_hw_tail_read() argument
418 static u32 gen7_oa_hw_tail_read(struct drm_i915_private *dev_priv) in gen7_oa_hw_tail_read() argument
449 static bool oa_buffer_check_unlocked(struct drm_i915_private *dev_priv) in oa_buffer_check_unlocked() argument
451 int report_size = dev_priv->perf.oa.oa_buffer.format_size; in oa_buffer_check_unlocked()
461 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
467 head = dev_priv->perf.oa.oa_buffer.head; in oa_buffer_check_unlocked()
469 aged_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; in oa_buffer_check_unlocked()
470 aged_tail = dev_priv->perf.oa.oa_buffer.tails[aged_idx].offset; in oa_buffer_check_unlocked()
471 aging_tail = dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset; in oa_buffer_check_unlocked()
473 hw_tail = dev_priv->perf.oa.ops.oa_hw_tail_read(dev_priv); in oa_buffer_check_unlocked()
493 ((now - dev_priv->perf.oa.oa_buffer.aging_timestamp) > in oa_buffer_check_unlocked()
497 dev_priv->perf.oa.oa_buffer.aged_tail_idx = aged_idx; in oa_buffer_check_unlocked()
502 dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR; in oa_buffer_check_unlocked()
517 struct i915_vma *vma = dev_priv->perf.oa.oa_buffer.vma; in oa_buffer_check_unlocked()
526 dev_priv->perf.oa.oa_buffer.tails[!aged_idx].offset = in oa_buffer_check_unlocked()
528 dev_priv->perf.oa.oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
535 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
598 struct drm_i915_private *dev_priv = stream->dev_priv; in append_oa_sample() local
599 int report_size = dev_priv->perf.oa.oa_buffer.format_size; in append_oa_sample()
650 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_append_oa_reports() local
651 int report_size = dev_priv->perf.oa.oa_buffer.format_size; in gen8_append_oa_reports()
652 u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; in gen8_append_oa_reports()
653 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); in gen8_append_oa_reports()
665 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
667 head = dev_priv->perf.oa.oa_buffer.head; in gen8_append_oa_reports()
668 aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; in gen8_append_oa_reports()
669 tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; in gen8_append_oa_reports()
671 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
735 if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) in gen8_append_oa_reports()
740 ctx_id = report32[2] & dev_priv->perf.oa.specific_ctx_id_mask; in gen8_append_oa_reports()
750 if (!(report32[0] & dev_priv->perf.oa.gen8_valid_ctx_bit)) in gen8_append_oa_reports()
784 if (!dev_priv->perf.oa.exclusive_stream->ctx || in gen8_append_oa_reports()
785 dev_priv->perf.oa.specific_ctx_id == ctx_id || in gen8_append_oa_reports()
786 (dev_priv->perf.oa.oa_buffer.last_ctx_id == in gen8_append_oa_reports()
787 dev_priv->perf.oa.specific_ctx_id) || in gen8_append_oa_reports()
794 if (dev_priv->perf.oa.exclusive_stream->ctx && in gen8_append_oa_reports()
795 dev_priv->perf.oa.specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
804 dev_priv->perf.oa.oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
818 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
827 dev_priv->perf.oa.oa_buffer.head = head; in gen8_append_oa_reports()
829 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
860 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_oa_read() local
864 if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) in gen8_oa_read()
890 dev_priv->perf.oa.period_exponent); in gen8_oa_read()
892 dev_priv->perf.oa.ops.oa_disable(dev_priv); in gen8_oa_read()
893 dev_priv->perf.oa.ops.oa_enable(dev_priv); in gen8_oa_read()
939 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_append_oa_reports() local
940 int report_size = dev_priv->perf.oa.oa_buffer.format_size; in gen7_append_oa_reports()
941 u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; in gen7_append_oa_reports()
942 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); in gen7_append_oa_reports()
954 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
956 head = dev_priv->perf.oa.oa_buffer.head; in gen7_append_oa_reports()
957 aged_tail_idx = dev_priv->perf.oa.oa_buffer.aged_tail_idx; in gen7_append_oa_reports()
958 tail = dev_priv->perf.oa.oa_buffer.tails[aged_tail_idx].offset; in gen7_append_oa_reports()
960 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1013 if (__ratelimit(&dev_priv->perf.oa.spurious_report_rs)) in gen7_append_oa_reports()
1032 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1042 dev_priv->perf.oa.oa_buffer.head = head; in gen7_append_oa_reports()
1044 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1071 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_oa_read() local
1075 if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) in gen7_oa_read()
1085 oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; in gen7_oa_read()
1114 dev_priv->perf.oa.period_exponent); in gen7_oa_read()
1116 dev_priv->perf.oa.ops.oa_disable(dev_priv); in gen7_oa_read()
1117 dev_priv->perf.oa.ops.oa_enable(dev_priv); in gen7_oa_read()
1127 dev_priv->perf.oa.gen7_latched_oastatus1 |= in gen7_oa_read()
1150 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_wait_unlocked() local
1153 if (!dev_priv->perf.oa.periodic) in i915_oa_wait_unlocked()
1156 return wait_event_interruptible(dev_priv->perf.oa.poll_wq, in i915_oa_wait_unlocked()
1157 oa_buffer_check_unlocked(dev_priv)); in i915_oa_wait_unlocked()
1174 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_poll_wait() local
1176 poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); in i915_oa_poll_wait()
1196 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_read() local
1198 return dev_priv->perf.oa.ops.read(stream, buf, count, offset); in i915_oa_read()
1240 struct drm_i915_private *i915 = stream->dev_priv; in oa_get_render_ctx_id()
1322 struct drm_i915_private *dev_priv = stream->dev_priv; in oa_put_render_ctx_id() local
1325 dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1326 dev_priv->perf.oa.specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1328 ce = fetch_and_zero(&dev_priv->perf.oa.pinned_ctx); in oa_put_render_ctx_id()
1330 mutex_lock(&dev_priv->drm.struct_mutex); in oa_put_render_ctx_id()
1332 mutex_unlock(&dev_priv->drm.struct_mutex); in oa_put_render_ctx_id()
1353 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_destroy() local
1355 BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); in i915_oa_stream_destroy()
1361 mutex_lock(&dev_priv->drm.struct_mutex); in i915_oa_stream_destroy()
1362 dev_priv->perf.oa.exclusive_stream = NULL; in i915_oa_stream_destroy()
1363 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); in i915_oa_stream_destroy()
1364 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_destroy()
1366 free_oa_buffer(dev_priv); in i915_oa_stream_destroy()
1368 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1369 intel_runtime_pm_put(dev_priv); in i915_oa_stream_destroy()
1374 put_oa_config(dev_priv, stream->oa_config); in i915_oa_stream_destroy()
1376 if (dev_priv->perf.oa.spurious_report_rs.missed) { in i915_oa_stream_destroy()
1378 dev_priv->perf.oa.spurious_report_rs.missed); in i915_oa_stream_destroy()
1382 static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) in gen7_init_oa_buffer() argument
1384 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); in gen7_init_oa_buffer()
1387 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1394 dev_priv->perf.oa.oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1401 dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1402 dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1404 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1410 dev_priv->perf.oa.gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1423 memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1428 dev_priv->perf.oa.pollin = false; in gen7_init_oa_buffer()
1431 static void gen8_init_oa_buffer(struct drm_i915_private *dev_priv) in gen8_init_oa_buffer() argument
1433 u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); in gen8_init_oa_buffer()
1436 spin_lock_irqsave(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1440 dev_priv->perf.oa.oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1457 dev_priv->perf.oa.oa_buffer.tails[0].offset = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1458 dev_priv->perf.oa.oa_buffer.tails[1].offset = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1465 dev_priv->perf.oa.oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1467 spin_unlock_irqrestore(&dev_priv->perf.oa.oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1481 memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1487 dev_priv->perf.oa.pollin = false; in gen8_init_oa_buffer()
1490 static int alloc_oa_buffer(struct drm_i915_private *dev_priv) in alloc_oa_buffer() argument
1496 if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) in alloc_oa_buffer()
1499 ret = i915_mutex_lock_interruptible(&dev_priv->drm); in alloc_oa_buffer()
1506 bo = i915_gem_object_create(dev_priv, OA_BUFFER_SIZE); in alloc_oa_buffer()
1523 dev_priv->perf.oa.oa_buffer.vma = vma; in alloc_oa_buffer()
1525 dev_priv->perf.oa.oa_buffer.vaddr = in alloc_oa_buffer()
1527 if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { in alloc_oa_buffer()
1528 ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); in alloc_oa_buffer()
1532 dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); in alloc_oa_buffer()
1535 i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), in alloc_oa_buffer()
1536 dev_priv->perf.oa.oa_buffer.vaddr); in alloc_oa_buffer()
1546 dev_priv->perf.oa.oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1547 dev_priv->perf.oa.oa_buffer.vma = NULL; in alloc_oa_buffer()
1550 mutex_unlock(&dev_priv->drm.struct_mutex); in alloc_oa_buffer()
1554 static void config_oa_regs(struct drm_i915_private *dev_priv, in config_oa_regs() argument
1567 static int hsw_enable_metric_set(struct drm_i915_private *dev_priv, in hsw_enable_metric_set() argument
1584 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); in hsw_enable_metric_set()
1609 config_oa_regs(dev_priv, oa_config->b_counter_regs, in hsw_enable_metric_set()
1615 static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) in hsw_disable_metric_set() argument
1637 struct drm_i915_private *dev_priv = ctx->i915; in gen8_update_reg_state_unlocked() local
1638 u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
1639 u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
1653 reg_state[ctx_oactxctrl+1] = (dev_priv->perf.oa.period_exponent << in gen8_update_reg_state_unlocked()
1655 (dev_priv->perf.oa.periodic ? in gen8_update_reg_state_unlocked()
1693 struct drm_i915_private *dev_priv = rq->i915; in gen8_emit_oa_config() local
1714 *cs++ = (dev_priv->perf.oa.period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_emit_oa_config()
1715 (dev_priv->perf.oa.periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_emit_oa_config()
1750 static int gen8_switch_to_updated_kernel_context(struct drm_i915_private *dev_priv, in gen8_switch_to_updated_kernel_context() argument
1753 struct intel_engine_cs *engine = dev_priv->engine[RCS]; in gen8_switch_to_updated_kernel_context()
1758 lockdep_assert_held(&dev_priv->drm.struct_mutex); in gen8_switch_to_updated_kernel_context()
1760 i915_retire_requests(dev_priv); in gen8_switch_to_updated_kernel_context()
1762 rq = i915_request_alloc(engine, dev_priv->kernel_context); in gen8_switch_to_updated_kernel_context()
1773 list_for_each_entry(timeline, &dev_priv->gt.timelines, link) { in gen8_switch_to_updated_kernel_context()
1777 &dev_priv->drm.struct_mutex); in gen8_switch_to_updated_kernel_context()
1811 static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv, in gen8_configure_all_contexts() argument
1814 struct intel_engine_cs *engine = dev_priv->engine[RCS]; in gen8_configure_all_contexts()
1819 lockdep_assert_held(&dev_priv->drm.struct_mutex); in gen8_configure_all_contexts()
1822 ret = gen8_switch_to_updated_kernel_context(dev_priv, oa_config); in gen8_configure_all_contexts()
1839 ret = i915_gem_wait_for_idle(dev_priv, in gen8_configure_all_contexts()
1846 list_for_each_entry(ctx, &dev_priv->contexts.list, link) { in gen8_configure_all_contexts()
1872 static int gen8_enable_metric_set(struct drm_i915_private *dev_priv, in gen8_enable_metric_set() argument
1900 if (IS_GEN(dev_priv, 9, 11)) { in gen8_enable_metric_set()
1911 ret = gen8_configure_all_contexts(dev_priv, oa_config); in gen8_enable_metric_set()
1915 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); in gen8_enable_metric_set()
1917 config_oa_regs(dev_priv, oa_config->b_counter_regs, in gen8_enable_metric_set()
1923 static void gen8_disable_metric_set(struct drm_i915_private *dev_priv) in gen8_disable_metric_set() argument
1926 gen8_configure_all_contexts(dev_priv, NULL); in gen8_disable_metric_set()
1932 static void gen10_disable_metric_set(struct drm_i915_private *dev_priv) in gen10_disable_metric_set() argument
1935 gen8_configure_all_contexts(dev_priv, NULL); in gen10_disable_metric_set()
1942 static void gen7_oa_enable(struct drm_i915_private *dev_priv) in gen7_oa_enable() argument
1945 dev_priv->perf.oa.exclusive_stream->ctx; in gen7_oa_enable()
1946 u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; in gen7_oa_enable()
1947 bool periodic = dev_priv->perf.oa.periodic; in gen7_oa_enable()
1948 u32 period_exponent = dev_priv->perf.oa.period_exponent; in gen7_oa_enable()
1949 u32 report_format = dev_priv->perf.oa.oa_buffer.format; in gen7_oa_enable()
1960 gen7_init_oa_buffer(dev_priv); in gen7_oa_enable()
1972 static void gen8_oa_enable(struct drm_i915_private *dev_priv) in gen8_oa_enable() argument
1974 u32 report_format = dev_priv->perf.oa.oa_buffer.format; in gen8_oa_enable()
1985 gen8_init_oa_buffer(dev_priv); in gen8_oa_enable()
2008 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_enable() local
2010 dev_priv->perf.oa.ops.oa_enable(dev_priv); in i915_oa_stream_enable()
2012 if (dev_priv->perf.oa.periodic) in i915_oa_stream_enable()
2013 hrtimer_start(&dev_priv->perf.oa.poll_check_timer, in i915_oa_stream_enable()
2018 static void gen7_oa_disable(struct drm_i915_private *dev_priv) in gen7_oa_disable() argument
2021 if (intel_wait_for_register(dev_priv, in gen7_oa_disable()
2027 static void gen8_oa_disable(struct drm_i915_private *dev_priv) in gen8_oa_disable() argument
2030 if (intel_wait_for_register(dev_priv, in gen8_oa_disable()
2046 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_disable() local
2048 dev_priv->perf.oa.ops.oa_disable(dev_priv); in i915_oa_stream_disable()
2050 if (dev_priv->perf.oa.periodic) in i915_oa_stream_disable()
2051 hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); in i915_oa_stream_disable()
2085 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_init() local
2093 if (!dev_priv->perf.metrics_kobj) { in i915_oa_stream_init()
2103 if (!dev_priv->perf.oa.ops.init_oa_buffer) { in i915_oa_stream_init()
2112 if (dev_priv->perf.oa.exclusive_stream) { in i915_oa_stream_init()
2133 ratelimit_state_init(&dev_priv->perf.oa.spurious_report_rs, in i915_oa_stream_init()
2139 ratelimit_set_flags(&dev_priv->perf.oa.spurious_report_rs, in i915_oa_stream_init()
2144 format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; in i915_oa_stream_init()
2149 dev_priv->perf.oa.oa_buffer.format_size = format_size; in i915_oa_stream_init()
2150 if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) in i915_oa_stream_init()
2153 dev_priv->perf.oa.oa_buffer.format = in i915_oa_stream_init()
2154 dev_priv->perf.oa.oa_formats[props->oa_format].format; in i915_oa_stream_init()
2156 dev_priv->perf.oa.periodic = props->oa_periodic; in i915_oa_stream_init()
2157 if (dev_priv->perf.oa.periodic) in i915_oa_stream_init()
2158 dev_priv->perf.oa.period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2168 ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config); in i915_oa_stream_init()
2186 intel_runtime_pm_get(dev_priv); in i915_oa_stream_init()
2187 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); in i915_oa_stream_init()
2189 ret = alloc_oa_buffer(dev_priv); in i915_oa_stream_init()
2193 ret = i915_mutex_lock_interruptible(&dev_priv->drm); in i915_oa_stream_init()
2197 ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv, in i915_oa_stream_init()
2206 dev_priv->perf.oa.exclusive_stream = stream; in i915_oa_stream_init()
2208 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_init()
2213 dev_priv->perf.oa.ops.disable_metric_set(dev_priv); in i915_oa_stream_init()
2214 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_init()
2217 free_oa_buffer(dev_priv); in i915_oa_stream_init()
2220 put_oa_config(dev_priv, stream->oa_config); in i915_oa_stream_init()
2222 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); in i915_oa_stream_init()
2223 intel_runtime_pm_put(dev_priv); in i915_oa_stream_init()
2313 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_read() local
2336 mutex_lock(&dev_priv->perf.lock); in i915_perf_read()
2339 mutex_unlock(&dev_priv->perf.lock); in i915_perf_read()
2342 mutex_lock(&dev_priv->perf.lock); in i915_perf_read()
2344 mutex_unlock(&dev_priv->perf.lock); in i915_perf_read()
2359 dev_priv->perf.oa.pollin = false; in i915_perf_read()
2367 struct drm_i915_private *dev_priv = in oa_poll_check_timer_cb() local
2368 container_of(hrtimer, typeof(*dev_priv), in oa_poll_check_timer_cb()
2371 if (oa_buffer_check_unlocked(dev_priv)) { in oa_poll_check_timer_cb()
2372 dev_priv->perf.oa.pollin = true; in oa_poll_check_timer_cb()
2373 wake_up(&dev_priv->perf.oa.poll_wq); in oa_poll_check_timer_cb()
2397 static __poll_t i915_perf_poll_locked(struct drm_i915_private *dev_priv, in i915_perf_poll_locked() argument
2412 if (dev_priv->perf.oa.pollin) in i915_perf_poll_locked()
2434 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_poll() local
2437 mutex_lock(&dev_priv->perf.lock); in i915_perf_poll()
2438 ret = i915_perf_poll_locked(dev_priv, stream, file, wait); in i915_perf_poll()
2439 mutex_unlock(&dev_priv->perf.lock); in i915_perf_poll()
2536 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_ioctl() local
2539 mutex_lock(&dev_priv->perf.lock); in i915_perf_ioctl()
2541 mutex_unlock(&dev_priv->perf.lock); in i915_perf_ioctl()
2586 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_release() local
2588 mutex_lock(&dev_priv->perf.lock); in i915_perf_release()
2590 mutex_unlock(&dev_priv->perf.lock); in i915_perf_release()
2635 i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, in i915_perf_open_ioctl_locked() argument
2674 if (IS_HASWELL(dev_priv) && specific_ctx) in i915_perf_open_ioctl_locked()
2695 stream->dev_priv = dev_priv; in i915_perf_open_ioctl_locked()
2711 list_add(&stream->link, &dev_priv->perf.streams); in i915_perf_open_ioctl_locked()
2743 static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) in oa_exponent_to_ns() argument
2746 1000ULL * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz); in oa_exponent_to_ns()
2764 static int read_properties_unlocked(struct drm_i915_private *dev_priv, in read_properties_unlocked() argument
2830 if (!dev_priv->perf.oa.oa_formats[value].size) { in read_properties_unlocked()
2851 oa_period = oa_exponent_to_ns(dev_priv, value); in read_properties_unlocked()
2914 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_open_ioctl() local
2920 if (!dev_priv->perf.initialized) { in i915_perf_open_ioctl()
2933 ret = read_properties_unlocked(dev_priv, in i915_perf_open_ioctl()
2940 mutex_lock(&dev_priv->perf.lock); in i915_perf_open_ioctl()
2941 ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); in i915_perf_open_ioctl()
2942 mutex_unlock(&dev_priv->perf.lock); in i915_perf_open_ioctl()
2955 void i915_perf_register(struct drm_i915_private *dev_priv) in i915_perf_register() argument
2959 if (!dev_priv->perf.initialized) in i915_perf_register()
2966 mutex_lock(&dev_priv->perf.lock); in i915_perf_register()
2968 dev_priv->perf.metrics_kobj = in i915_perf_register()
2970 &dev_priv->drm.primary->kdev->kobj); in i915_perf_register()
2971 if (!dev_priv->perf.metrics_kobj) in i915_perf_register()
2974 sysfs_attr_init(&dev_priv->perf.oa.test_config.sysfs_metric_id.attr); in i915_perf_register()
2976 if (IS_HASWELL(dev_priv)) { in i915_perf_register()
2977 i915_perf_load_test_config_hsw(dev_priv); in i915_perf_register()
2978 } else if (IS_BROADWELL(dev_priv)) { in i915_perf_register()
2979 i915_perf_load_test_config_bdw(dev_priv); in i915_perf_register()
2980 } else if (IS_CHERRYVIEW(dev_priv)) { in i915_perf_register()
2981 i915_perf_load_test_config_chv(dev_priv); in i915_perf_register()
2982 } else if (IS_SKYLAKE(dev_priv)) { in i915_perf_register()
2983 if (IS_SKL_GT2(dev_priv)) in i915_perf_register()
2984 i915_perf_load_test_config_sklgt2(dev_priv); in i915_perf_register()
2985 else if (IS_SKL_GT3(dev_priv)) in i915_perf_register()
2986 i915_perf_load_test_config_sklgt3(dev_priv); in i915_perf_register()
2987 else if (IS_SKL_GT4(dev_priv)) in i915_perf_register()
2988 i915_perf_load_test_config_sklgt4(dev_priv); in i915_perf_register()
2989 } else if (IS_BROXTON(dev_priv)) { in i915_perf_register()
2990 i915_perf_load_test_config_bxt(dev_priv); in i915_perf_register()
2991 } else if (IS_KABYLAKE(dev_priv)) { in i915_perf_register()
2992 if (IS_KBL_GT2(dev_priv)) in i915_perf_register()
2993 i915_perf_load_test_config_kblgt2(dev_priv); in i915_perf_register()
2994 else if (IS_KBL_GT3(dev_priv)) in i915_perf_register()
2995 i915_perf_load_test_config_kblgt3(dev_priv); in i915_perf_register()
2996 } else if (IS_GEMINILAKE(dev_priv)) { in i915_perf_register()
2997 i915_perf_load_test_config_glk(dev_priv); in i915_perf_register()
2998 } else if (IS_COFFEELAKE(dev_priv)) { in i915_perf_register()
2999 if (IS_CFL_GT2(dev_priv)) in i915_perf_register()
3000 i915_perf_load_test_config_cflgt2(dev_priv); in i915_perf_register()
3001 if (IS_CFL_GT3(dev_priv)) in i915_perf_register()
3002 i915_perf_load_test_config_cflgt3(dev_priv); in i915_perf_register()
3003 } else if (IS_CANNONLAKE(dev_priv)) { in i915_perf_register()
3004 i915_perf_load_test_config_cnl(dev_priv); in i915_perf_register()
3005 } else if (IS_ICELAKE(dev_priv)) { in i915_perf_register()
3006 i915_perf_load_test_config_icl(dev_priv); in i915_perf_register()
3009 if (dev_priv->perf.oa.test_config.id == 0) in i915_perf_register()
3012 ret = sysfs_create_group(dev_priv->perf.metrics_kobj, in i915_perf_register()
3013 &dev_priv->perf.oa.test_config.sysfs_metric); in i915_perf_register()
3017 atomic_set(&dev_priv->perf.oa.test_config.ref_count, 1); in i915_perf_register()
3022 kobject_put(dev_priv->perf.metrics_kobj); in i915_perf_register()
3023 dev_priv->perf.metrics_kobj = NULL; in i915_perf_register()
3026 mutex_unlock(&dev_priv->perf.lock); in i915_perf_register()
3038 void i915_perf_unregister(struct drm_i915_private *dev_priv) in i915_perf_unregister() argument
3040 if (!dev_priv->perf.metrics_kobj) in i915_perf_unregister()
3043 sysfs_remove_group(dev_priv->perf.metrics_kobj, in i915_perf_unregister()
3044 &dev_priv->perf.oa.test_config.sysfs_metric); in i915_perf_unregister()
3046 kobject_put(dev_priv->perf.metrics_kobj); in i915_perf_unregister()
3047 dev_priv->perf.metrics_kobj = NULL; in i915_perf_unregister()
3050 static bool gen8_is_valid_flex_addr(struct drm_i915_private *dev_priv, u32 addr) in gen8_is_valid_flex_addr() argument
3070 static bool gen7_is_valid_b_counter_addr(struct drm_i915_private *dev_priv, u32 addr) in gen7_is_valid_b_counter_addr() argument
3080 static bool gen7_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) in gen7_is_valid_mux_addr() argument
3091 static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) in gen8_is_valid_mux_addr() argument
3093 return gen7_is_valid_mux_addr(dev_priv, addr) || in gen8_is_valid_mux_addr()
3099 static bool gen10_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) in gen10_is_valid_mux_addr() argument
3101 return gen8_is_valid_mux_addr(dev_priv, addr) || in gen10_is_valid_mux_addr()
3106 static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) in hsw_is_valid_mux_addr() argument
3108 return gen7_is_valid_mux_addr(dev_priv, addr) || in hsw_is_valid_mux_addr()
3115 static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr) in chv_is_valid_mux_addr() argument
3117 return gen7_is_valid_mux_addr(dev_priv, addr) || in chv_is_valid_mux_addr()
3140 static struct i915_oa_reg *alloc_oa_regs(struct drm_i915_private *dev_priv, in alloc_oa_regs() argument
3141 bool (*is_valid)(struct drm_i915_private *dev_priv, u32 addr), in alloc_oa_regs() argument
3171 if (!is_valid(dev_priv, addr)) { in alloc_oa_regs()
3204 static int create_dynamic_oa_sysfs_entry(struct drm_i915_private *dev_priv, in create_dynamic_oa_sysfs_entry() argument
3219 return sysfs_create_group(dev_priv->perf.metrics_kobj, in create_dynamic_oa_sysfs_entry()
3239 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_add_config_ioctl() local
3244 if (!dev_priv->perf.initialized) { in i915_perf_add_config_ioctl()
3249 if (!dev_priv->perf.metrics_kobj) { in i915_perf_add_config_ioctl()
3287 alloc_oa_regs(dev_priv, in i915_perf_add_config_ioctl()
3288 dev_priv->perf.oa.ops.is_valid_mux_reg, in i915_perf_add_config_ioctl()
3300 alloc_oa_regs(dev_priv, in i915_perf_add_config_ioctl()
3301 dev_priv->perf.oa.ops.is_valid_b_counter_reg, in i915_perf_add_config_ioctl()
3311 if (INTEL_GEN(dev_priv) < 8) { in i915_perf_add_config_ioctl()
3319 alloc_oa_regs(dev_priv, in i915_perf_add_config_ioctl()
3320 dev_priv->perf.oa.ops.is_valid_flex_reg, in i915_perf_add_config_ioctl()
3331 err = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3338 idr_for_each_entry(&dev_priv->perf.metrics_idr, tmp, id) { in i915_perf_add_config_ioctl()
3346 err = create_dynamic_oa_sysfs_entry(dev_priv, oa_config); in i915_perf_add_config_ioctl()
3353 oa_config->id = idr_alloc(&dev_priv->perf.metrics_idr, in i915_perf_add_config_ioctl()
3362 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3369 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3371 put_oa_config(dev_priv, oa_config); in i915_perf_add_config_ioctl()
3390 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_remove_config_ioctl() local
3395 if (!dev_priv->perf.initialized) { in i915_perf_remove_config_ioctl()
3405 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in i915_perf_remove_config_ioctl()
3409 oa_config = idr_find(&dev_priv->perf.metrics_idr, *arg); in i915_perf_remove_config_ioctl()
3418 sysfs_remove_group(dev_priv->perf.metrics_kobj, in i915_perf_remove_config_ioctl()
3421 idr_remove(&dev_priv->perf.metrics_idr, *arg); in i915_perf_remove_config_ioctl()
3425 put_oa_config(dev_priv, oa_config); in i915_perf_remove_config_ioctl()
3428 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_remove_config_ioctl()
3484 void i915_perf_init(struct drm_i915_private *dev_priv) in i915_perf_init() argument
3486 if (IS_HASWELL(dev_priv)) { in i915_perf_init()
3487 dev_priv->perf.oa.ops.is_valid_b_counter_reg = in i915_perf_init()
3489 dev_priv->perf.oa.ops.is_valid_mux_reg = in i915_perf_init()
3491 dev_priv->perf.oa.ops.is_valid_flex_reg = NULL; in i915_perf_init()
3492 dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; in i915_perf_init()
3493 dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; in i915_perf_init()
3494 dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; in i915_perf_init()
3495 dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; in i915_perf_init()
3496 dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; in i915_perf_init()
3497 dev_priv->perf.oa.ops.read = gen7_oa_read; in i915_perf_init()
3498 dev_priv->perf.oa.ops.oa_hw_tail_read = in i915_perf_init()
3501 dev_priv->perf.oa.oa_formats = hsw_oa_formats; in i915_perf_init()
3502 } else if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) { in i915_perf_init()
3509 dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats; in i915_perf_init()
3511 dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer; in i915_perf_init()
3512 dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable; in i915_perf_init()
3513 dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable; in i915_perf_init()
3514 dev_priv->perf.oa.ops.read = gen8_oa_read; in i915_perf_init()
3515 dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
3517 if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) { in i915_perf_init()
3518 dev_priv->perf.oa.ops.is_valid_b_counter_reg = in i915_perf_init()
3520 dev_priv->perf.oa.ops.is_valid_mux_reg = in i915_perf_init()
3522 dev_priv->perf.oa.ops.is_valid_flex_reg = in i915_perf_init()
3525 if (IS_CHERRYVIEW(dev_priv)) { in i915_perf_init()
3526 dev_priv->perf.oa.ops.is_valid_mux_reg = in i915_perf_init()
3530 dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
3531 dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set; in i915_perf_init()
3533 if (IS_GEN8(dev_priv)) { in i915_perf_init()
3534 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120; in i915_perf_init()
3535 dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce; in i915_perf_init()
3537 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25); in i915_perf_init()
3539 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; in i915_perf_init()
3540 dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; in i915_perf_init()
3542 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); in i915_perf_init()
3544 } else if (IS_GEN(dev_priv, 10, 11)) { in i915_perf_init()
3545 dev_priv->perf.oa.ops.is_valid_b_counter_reg = in i915_perf_init()
3547 dev_priv->perf.oa.ops.is_valid_mux_reg = in i915_perf_init()
3549 dev_priv->perf.oa.ops.is_valid_flex_reg = in i915_perf_init()
3552 dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
3553 dev_priv->perf.oa.ops.disable_metric_set = gen10_disable_metric_set; in i915_perf_init()
3555 dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128; in i915_perf_init()
3556 dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de; in i915_perf_init()
3558 dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16); in i915_perf_init()
3562 if (dev_priv->perf.oa.ops.enable_metric_set) { in i915_perf_init()
3563 hrtimer_init(&dev_priv->perf.oa.poll_check_timer, in i915_perf_init()
3565 dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; in i915_perf_init()
3566 init_waitqueue_head(&dev_priv->perf.oa.poll_wq); in i915_perf_init()
3568 INIT_LIST_HEAD(&dev_priv->perf.streams); in i915_perf_init()
3569 mutex_init(&dev_priv->perf.lock); in i915_perf_init()
3570 spin_lock_init(&dev_priv->perf.oa.oa_buffer.ptr_lock); in i915_perf_init()
3573 (INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); in i915_perf_init()
3574 dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); in i915_perf_init()
3576 mutex_init(&dev_priv->perf.metrics_lock); in i915_perf_init()
3577 idr_init(&dev_priv->perf.metrics_idr); in i915_perf_init()
3579 dev_priv->perf.initialized = true; in i915_perf_init()
3585 struct drm_i915_private *dev_priv = data; in destroy_config() local
3588 put_oa_config(dev_priv, oa_config); in destroy_config()
3597 void i915_perf_fini(struct drm_i915_private *dev_priv) in i915_perf_fini() argument
3599 if (!dev_priv->perf.initialized) in i915_perf_fini()
3602 idr_for_each(&dev_priv->perf.metrics_idr, destroy_config, dev_priv); in i915_perf_fini()
3603 idr_destroy(&dev_priv->perf.metrics_idr); in i915_perf_fini()
3605 unregister_sysctl_table(dev_priv->perf.sysctl_header); in i915_perf_fini()
3607 memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); in i915_perf_fini()
3609 dev_priv->perf.initialized = false; in i915_perf_fini()