Lines Matching +full:cpu +full:- +full:centric
2 * Copyright © 2015-2016 Intel Corporation
40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
44 * without special privileges. Access to system-wide metrics requires root
58 * might sample sets of tightly-coupled counters, depending on the
70 * interleaved with event-type specific members.
74 * with the CPU, using HW specific packing formats for counter sets. Sometimes
76 * would be acceptable to expose them to unprivileged applications - to hide
88 * into perf's currently cpu centric design.
96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
102 * For posterity, in case we might re-visit trying to adapt core perf to be
106 * - The perf based OA PMU driver broke some significant design assumptions:
108 * Existing perf pmus are used for profiling work on a cpu and we were
110 * implications, the need to fake cpu-related data (such as user/kernel
112 * as a way to forward device-specific status records.
115 * involvement from the CPU, making our PMU driver the first of a kind.
117 * Given the way we were periodically forward data from the GPU-mapped, OA
123 * explicitly initiated from the cpu (say in response to a userspace read())
125 * trigger a report from the cpu on demand.
130 * opened, there's no clear precedent for being able to provide group-wide
139 * for combining with the side-band raw reports it captures using
142 * - As a side note on perf's grouping feature; there was also some concern
158 * one time. The OA unit is not designed to allow re-configuration while in
178 * - It felt like our perf based PMU was making some technical compromises
182 * cpu core, while our device pmu related to neither. Events opened with a
184 * that process - so not appropriate for us. When an event is related to a
185 * cpu id, perf ensures pmu methods will be invoked via an inter process
187 * perf events for a specific cpu. This was workable but it meant the
225 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
233 * CPU).
236 * by checking for a zeroed report-id field in tail reports, we want to account
242 * time for the corresponding reports to become visible to the CPU.
246 * can trust the corresponding data is visible to the CPU; at which point
265 * non-periodic reports (such as on context switch) or the OA unit may be
318 * code assumes all reports have a power-of-two size and ~(size - 1) can
342 * struct perf_open_properties - for validated properties given to open a stream
373 if (!PTR_ERR(oa_config->flex_regs)) in free_oa_config()
374 kfree(oa_config->flex_regs); in free_oa_config()
375 if (!PTR_ERR(oa_config->b_counter_regs)) in free_oa_config()
376 kfree(oa_config->b_counter_regs); in free_oa_config()
377 if (!PTR_ERR(oa_config->mux_regs)) in free_oa_config()
378 kfree(oa_config->mux_regs); in free_oa_config()
385 if (!atomic_dec_and_test(&oa_config->ref_count)) in put_oa_config()
398 *out_config = &dev_priv->perf.test_config; in get_oa_config()
399 atomic_inc(&dev_priv->perf.test_config.ref_count); in get_oa_config()
403 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in get_oa_config()
407 *out_config = idr_find(&dev_priv->perf.metrics_idr, metrics_set); in get_oa_config()
409 ret = -EINVAL; in get_oa_config()
411 atomic_inc(&(*out_config)->ref_count); in get_oa_config()
413 mutex_unlock(&dev_priv->perf.metrics_lock); in get_oa_config()
420 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_oa_hw_tail_read()
427 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_oa_hw_tail_read()
434 * oa_buffer_check_unlocked - check for data and update tail ptr state
442 * pointer having a race with respect to what data is visible to the CPU.
459 struct drm_i915_private *dev_priv = stream->dev_priv; in oa_buffer_check_unlocked()
460 int report_size = stream->oa_buffer.format_size; in oa_buffer_check_unlocked()
470 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
476 head = stream->oa_buffer.head; in oa_buffer_check_unlocked()
478 aged_idx = stream->oa_buffer.aged_tail_idx; in oa_buffer_check_unlocked()
479 aged_tail = stream->oa_buffer.tails[aged_idx].offset; in oa_buffer_check_unlocked()
480 aging_tail = stream->oa_buffer.tails[!aged_idx].offset; in oa_buffer_check_unlocked()
482 hw_tail = dev_priv->perf.ops.oa_hw_tail_read(stream); in oa_buffer_check_unlocked()
487 hw_tail &= ~(report_size - 1); in oa_buffer_check_unlocked()
495 * the CPU... in oa_buffer_check_unlocked()
502 ((now - stream->oa_buffer.aging_timestamp) > in oa_buffer_check_unlocked()
506 stream->oa_buffer.aged_tail_idx = aged_idx; in oa_buffer_check_unlocked()
511 stream->oa_buffer.tails[!aged_idx].offset = INVALID_TAIL_PTR; in oa_buffer_check_unlocked()
526 struct i915_vma *vma = stream->oa_buffer.vma; in oa_buffer_check_unlocked()
535 stream->oa_buffer.tails[!aged_idx].offset = in oa_buffer_check_unlocked()
537 stream->oa_buffer.aging_timestamp = now; in oa_buffer_check_unlocked()
544 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in oa_buffer_check_unlocked()
551 * append_oa_status - Appends a status record to a userspace read() buffer.
552 * @stream: An i915-perf stream opened for OA metrics
573 if ((count - *offset) < header.size) in append_oa_status()
574 return -ENOSPC; in append_oa_status()
577 return -EFAULT; in append_oa_status()
585 * append_oa_sample - Copies single OA report into userspace read() buffer.
586 * @stream: An i915-perf stream opened for OA metrics
593 * properties when opening a stream, tracked as `stream->sample_flags`. This
607 int report_size = stream->oa_buffer.format_size; in append_oa_sample()
609 u32 sample_flags = stream->sample_flags; in append_oa_sample()
613 header.size = stream->sample_size; in append_oa_sample()
615 if ((count - *offset) < header.size) in append_oa_sample()
616 return -ENOSPC; in append_oa_sample()
620 return -EFAULT; in append_oa_sample()
625 return -EFAULT; in append_oa_sample()
635 * @stream: An i915-perf stream opened for OA metrics
640 * Notably any error condition resulting in a short read (-%ENOSPC or
641 * -%EFAULT) will be returned even though one or more records may
648 * and back-to-front you're not alone, but this follows the
658 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_append_oa_reports()
659 int report_size = stream->oa_buffer.format_size; in gen8_append_oa_reports()
660 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen8_append_oa_reports()
661 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_append_oa_reports()
662 u32 mask = (OA_BUFFER_SIZE - 1); in gen8_append_oa_reports()
670 if (WARN_ON(!stream->enabled)) in gen8_append_oa_reports()
671 return -EIO; in gen8_append_oa_reports()
673 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
675 head = stream->oa_buffer.head; in gen8_append_oa_reports()
676 aged_tail_idx = stream->oa_buffer.aged_tail_idx; in gen8_append_oa_reports()
677 tail = stream->oa_buffer.tails[aged_tail_idx].offset; in gen8_append_oa_reports()
679 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
686 return -EAGAIN; in gen8_append_oa_reports()
692 head -= gtt_offset; in gen8_append_oa_reports()
693 tail -= gtt_offset; in gen8_append_oa_reports()
706 return -EIO; in gen8_append_oa_reports()
726 if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { in gen8_append_oa_reports()
727 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); in gen8_append_oa_reports()
743 if (__ratelimit(&dev_priv->perf.spurious_report_rs)) in gen8_append_oa_reports()
748 ctx_id = report32[2] & stream->specific_ctx_id_mask; in gen8_append_oa_reports()
752 * invalid to be sure we avoid false-positive, single-context in gen8_append_oa_reports()
758 if (!(report32[0] & dev_priv->perf.gen8_valid_ctx_bit)) in gen8_append_oa_reports()
764 * stop the counters from updating as system-wide / global in gen8_append_oa_reports()
768 * filtered on the cpu but it's not worth trying to in gen8_append_oa_reports()
772 * provide a side-band view of the real values. in gen8_append_oa_reports()
776 * needs be forwarded bookend context-switch reports so that it in gen8_append_oa_reports()
789 * switches since it's not-uncommon for periodic samples to in gen8_append_oa_reports()
792 if (!dev_priv->perf.exclusive_stream->ctx || in gen8_append_oa_reports()
793 stream->specific_ctx_id == ctx_id || in gen8_append_oa_reports()
794 stream->oa_buffer.last_ctx_id == stream->specific_ctx_id || in gen8_append_oa_reports()
801 if (dev_priv->perf.exclusive_stream->ctx && in gen8_append_oa_reports()
802 stream->specific_ctx_id != ctx_id) { in gen8_append_oa_reports()
811 stream->oa_buffer.last_ctx_id = ctx_id; in gen8_append_oa_reports()
825 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
834 stream->oa_buffer.head = head; in gen8_append_oa_reports()
836 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_append_oa_reports()
843 * gen8_oa_read - copy status records then buffered OA reports
844 * @stream: An i915-perf stream opened for OA metrics
867 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_oa_read()
871 if (WARN_ON(!stream->oa_buffer.vaddr)) in gen8_oa_read()
872 return -EIO; in gen8_oa_read()
897 stream->period_exponent); in gen8_oa_read()
899 dev_priv->perf.ops.oa_disable(stream); in gen8_oa_read()
900 dev_priv->perf.ops.oa_enable(stream); in gen8_oa_read()
903 * Note: .oa_enable() is expected to re-init the oabuffer and in gen8_oa_read()
923 * @stream: An i915-perf stream opened for OA metrics
928 * Notably any error condition resulting in a short read (-%ENOSPC or
929 * -%EFAULT) will be returned even though one or more records may
936 * and back-to-front you're not alone, but this follows the
946 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_append_oa_reports()
947 int report_size = stream->oa_buffer.format_size; in gen7_append_oa_reports()
948 u8 *oa_buf_base = stream->oa_buffer.vaddr; in gen7_append_oa_reports()
949 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_append_oa_reports()
950 u32 mask = (OA_BUFFER_SIZE - 1); in gen7_append_oa_reports()
958 if (WARN_ON(!stream->enabled)) in gen7_append_oa_reports()
959 return -EIO; in gen7_append_oa_reports()
961 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
963 head = stream->oa_buffer.head; in gen7_append_oa_reports()
964 aged_tail_idx = stream->oa_buffer.aged_tail_idx; in gen7_append_oa_reports()
965 tail = stream->oa_buffer.tails[aged_tail_idx].offset; in gen7_append_oa_reports()
967 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
973 return -EAGAIN; in gen7_append_oa_reports()
978 head -= gtt_offset; in gen7_append_oa_reports()
979 tail -= gtt_offset; in gen7_append_oa_reports()
991 return -EIO; in gen7_append_oa_reports()
1008 if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { in gen7_append_oa_reports()
1009 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); in gen7_append_oa_reports()
1013 /* The report-ID field for periodic samples includes in gen7_append_oa_reports()
1020 if (__ratelimit(&dev_priv->perf.spurious_report_rs)) in gen7_append_oa_reports()
1029 /* The above report-id field sanity check is based on in gen7_append_oa_reports()
1039 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1049 stream->oa_buffer.head = head; in gen7_append_oa_reports()
1051 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_append_oa_reports()
1058 * gen7_oa_read - copy status records then buffered OA reports
1059 * @stream: An i915-perf stream opened for OA metrics
1078 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_oa_read()
1082 if (WARN_ON(!stream->oa_buffer.vaddr)) in gen7_oa_read()
1083 return -EIO; in gen7_oa_read()
1092 oastatus1 &= ~dev_priv->perf.gen7_latched_oastatus1; in gen7_oa_read()
1096 * - The status can be interpreted to mean that the buffer is in gen7_oa_read()
1098 * which will start to report a near-empty buffer after an in gen7_oa_read()
1103 * - Since it also implies the HW has started overwriting old in gen7_oa_read()
1108 * - In the future we may want to introduce a flight recorder in gen7_oa_read()
1121 stream->period_exponent); in gen7_oa_read()
1123 dev_priv->perf.ops.oa_disable(stream); in gen7_oa_read()
1124 dev_priv->perf.ops.oa_enable(stream); in gen7_oa_read()
1134 dev_priv->perf.gen7_latched_oastatus1 |= in gen7_oa_read()
1142 * i915_oa_wait_unlocked - handles blocking IO until OA data available
1143 * @stream: An i915-perf stream opened for OA metrics
1146 * for OA metrics. It waits until the hrtimer callback finds a non-empty
1150 * since any subsequent read handling will return -EAGAIN if there isn't
1158 if (!stream->periodic) in i915_oa_wait_unlocked()
1159 return -EIO; in i915_oa_wait_unlocked()
1161 return wait_event_interruptible(stream->poll_wq, in i915_oa_wait_unlocked()
1166 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1167 * @stream: An i915-perf stream opened for OA metrics
1179 poll_wait(file, &stream->poll_wq, wait); in i915_oa_poll_wait()
1183 * i915_oa_read - just calls through to &i915_oa_ops->read
1184 * @stream: An i915-perf stream opened for OA metrics
1199 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_read()
1201 return dev_priv->perf.ops.read(stream, buf, count, offset); in i915_oa_read()
1207 struct drm_i915_private *i915 = stream->dev_priv; in oa_pin_context()
1208 struct i915_gem_context *ctx = stream->ctx; in oa_pin_context()
1212 err = i915_mutex_lock_interruptible(&i915->drm); in oa_pin_context()
1217 if (ce->engine->class != RENDER_CLASS) in oa_pin_context()
1226 stream->pinned_ctx = ce; in oa_pin_context()
1232 mutex_unlock(&i915->drm.struct_mutex); in oa_pin_context()
1236 return stream->pinned_ctx; in oa_pin_context()
1240 * oa_get_render_ctx_id - determine and hold ctx hw id
1241 * @stream: An i915-perf stream opened for OA metrics
1251 struct drm_i915_private *i915 = stream->dev_priv; in oa_get_render_ctx_id()
1264 stream->specific_ctx_id = i915_ggtt_offset(ce->state); in oa_get_render_ctx_id()
1265 stream->specific_ctx_id_mask = 0; in oa_get_render_ctx_id()
1283 stream->specific_ctx_id = in oa_get_render_ctx_id()
1284 lower_32_bits(ce->lrc_desc) >> 12; in oa_get_render_ctx_id()
1290 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1291 (1U << (GEN8_CTX_ID_WIDTH - 1)) - 1; in oa_get_render_ctx_id()
1293 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1294 (1U << GEN8_CTX_ID_WIDTH) - 1; in oa_get_render_ctx_id()
1295 stream->specific_ctx_id = in oa_get_render_ctx_id()
1296 upper_32_bits(ce->lrc_desc); in oa_get_render_ctx_id()
1297 stream->specific_ctx_id &= in oa_get_render_ctx_id()
1298 stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1303 stream->specific_ctx_id_mask = in oa_get_render_ctx_id()
1304 ((1U << GEN11_SW_CTX_ID_WIDTH) - 1) << (GEN11_SW_CTX_ID_SHIFT - 32) | in oa_get_render_ctx_id()
1305 ((1U << GEN11_ENGINE_INSTANCE_WIDTH) - 1) << (GEN11_ENGINE_INSTANCE_SHIFT - 32) | in oa_get_render_ctx_id()
1306 ((1 << GEN11_ENGINE_CLASS_WIDTH) - 1) << (GEN11_ENGINE_CLASS_SHIFT - 32); in oa_get_render_ctx_id()
1307 stream->specific_ctx_id = upper_32_bits(ce->lrc_desc); in oa_get_render_ctx_id()
1308 stream->specific_ctx_id &= in oa_get_render_ctx_id()
1309 stream->specific_ctx_id_mask; in oa_get_render_ctx_id()
1318 stream->specific_ctx_id, in oa_get_render_ctx_id()
1319 stream->specific_ctx_id_mask); in oa_get_render_ctx_id()
1325 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1326 * @stream: An i915-perf stream opened for OA metrics
1333 struct drm_i915_private *dev_priv = stream->dev_priv; in oa_put_render_ctx_id()
1336 stream->specific_ctx_id = INVALID_CTX_ID; in oa_put_render_ctx_id()
1337 stream->specific_ctx_id_mask = 0; in oa_put_render_ctx_id()
1339 ce = fetch_and_zero(&stream->pinned_ctx); in oa_put_render_ctx_id()
1341 mutex_lock(&dev_priv->drm.struct_mutex); in oa_put_render_ctx_id()
1343 mutex_unlock(&dev_priv->drm.struct_mutex); in oa_put_render_ctx_id()
1350 struct drm_i915_private *i915 = stream->dev_priv; in free_oa_buffer()
1352 mutex_lock(&i915->drm.struct_mutex); in free_oa_buffer()
1354 i915_vma_unpin_and_release(&stream->oa_buffer.vma, in free_oa_buffer()
1357 mutex_unlock(&i915->drm.struct_mutex); in free_oa_buffer()
1359 stream->oa_buffer.vaddr = NULL; in free_oa_buffer()
1364 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_destroy()
1366 BUG_ON(stream != dev_priv->perf.exclusive_stream); in i915_oa_stream_destroy()
1372 mutex_lock(&dev_priv->drm.struct_mutex); in i915_oa_stream_destroy()
1373 dev_priv->perf.exclusive_stream = NULL; in i915_oa_stream_destroy()
1374 dev_priv->perf.ops.disable_metric_set(stream); in i915_oa_stream_destroy()
1375 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_destroy()
1379 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); in i915_oa_stream_destroy()
1380 intel_runtime_pm_put(&dev_priv->runtime_pm, stream->wakeref); in i915_oa_stream_destroy()
1382 if (stream->ctx) in i915_oa_stream_destroy()
1385 put_oa_config(dev_priv, stream->oa_config); in i915_oa_stream_destroy()
1387 if (dev_priv->perf.spurious_report_rs.missed) { in i915_oa_stream_destroy()
1389 dev_priv->perf.spurious_report_rs.missed); in i915_oa_stream_destroy()
1395 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_init_oa_buffer()
1396 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen7_init_oa_buffer()
1399 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1401 /* Pre-DevBDW: OABUFFER must be set with counters off, in gen7_init_oa_buffer()
1406 stream->oa_buffer.head = gtt_offset; in gen7_init_oa_buffer()
1413 stream->oa_buffer.tails[0].offset = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1414 stream->oa_buffer.tails[1].offset = INVALID_TAIL_PTR; in gen7_init_oa_buffer()
1416 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen7_init_oa_buffer()
1422 dev_priv->perf.gen7_latched_oastatus1 = 0; in gen7_init_oa_buffer()
1426 * first allocating), we may re-init the OA buffer, either in gen7_init_oa_buffer()
1427 * when re-enabling a stream or in error/reset paths. in gen7_init_oa_buffer()
1429 * The reason we clear the buffer for each re-init is for the in gen7_init_oa_buffer()
1431 * report-id field to make sure it's non-zero which relies on in gen7_init_oa_buffer()
1435 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen7_init_oa_buffer()
1437 /* Maybe make ->pollin per-stream state if we support multiple in gen7_init_oa_buffer()
1440 stream->pollin = false; in gen7_init_oa_buffer()
1445 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_init_oa_buffer()
1446 u32 gtt_offset = i915_ggtt_offset(stream->oa_buffer.vma); in gen8_init_oa_buffer()
1449 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1453 stream->oa_buffer.head = gtt_offset; in gen8_init_oa_buffer()
1470 stream->oa_buffer.tails[0].offset = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1471 stream->oa_buffer.tails[1].offset = INVALID_TAIL_PTR; in gen8_init_oa_buffer()
1478 stream->oa_buffer.last_ctx_id = INVALID_CTX_ID; in gen8_init_oa_buffer()
1480 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in gen8_init_oa_buffer()
1485 * first allocating), we may re-init the OA buffer, either in gen8_init_oa_buffer()
1486 * when re-enabling a stream or in error/reset paths. in gen8_init_oa_buffer()
1488 * The reason we clear the buffer for each re-init is for the in gen8_init_oa_buffer()
1490 * reason field to make sure it's non-zero which relies on in gen8_init_oa_buffer()
1494 memset(stream->oa_buffer.vaddr, 0, OA_BUFFER_SIZE); in gen8_init_oa_buffer()
1497 * Maybe make ->pollin per-stream state if we support multiple in gen8_init_oa_buffer()
1500 stream->pollin = false; in gen8_init_oa_buffer()
1506 struct drm_i915_private *dev_priv = stream->dev_priv; in alloc_oa_buffer()
1510 if (WARN_ON(stream->oa_buffer.vma)) in alloc_oa_buffer()
1511 return -ENODEV; in alloc_oa_buffer()
1513 ret = i915_mutex_lock_interruptible(&dev_priv->drm); in alloc_oa_buffer()
1535 stream->oa_buffer.vma = vma; in alloc_oa_buffer()
1537 stream->oa_buffer.vaddr = in alloc_oa_buffer()
1539 if (IS_ERR(stream->oa_buffer.vaddr)) { in alloc_oa_buffer()
1540 ret = PTR_ERR(stream->oa_buffer.vaddr); in alloc_oa_buffer()
1545 i915_ggtt_offset(stream->oa_buffer.vma), in alloc_oa_buffer()
1546 stream->oa_buffer.vaddr); in alloc_oa_buffer()
1556 stream->oa_buffer.vaddr = NULL; in alloc_oa_buffer()
1557 stream->oa_buffer.vma = NULL; in alloc_oa_buffer()
1560 mutex_unlock(&dev_priv->drm.struct_mutex); in alloc_oa_buffer()
1573 I915_WRITE(reg->addr, reg->value); in config_oa_regs()
1588 * generated before this config has completed - albeit not in delay_after_mux()
1598 * problem - it just seems like the simplest explanation why in delay_after_mux()
1606 struct drm_i915_private *dev_priv = stream->dev_priv; in hsw_enable_metric_set()
1607 const struct i915_oa_config *oa_config = stream->oa_config; in hsw_enable_metric_set()
1614 * unable to count the events from non-render clock domain. in hsw_enable_metric_set()
1616 * count the events from non-render domain. Unit level clock in hsw_enable_metric_set()
1624 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); in hsw_enable_metric_set()
1627 config_oa_regs(dev_priv, oa_config->b_counter_regs, in hsw_enable_metric_set()
1628 oa_config->b_counter_regs_len); in hsw_enable_metric_set()
1635 struct drm_i915_private *dev_priv = stream->dev_priv; in hsw_disable_metric_set()
1660 for (i = 0; i < oa_config->flex_regs_len; i++) { in oa_config_flex_reg()
1661 if (i915_mmio_reg_offset(oa_config->flex_regs[i].addr) == mmio) in oa_config_flex_reg()
1662 return oa_config->flex_regs[i].value; in oa_config_flex_reg()
1671 * It's fine to put out-of-date values into these per-context registers
1680 struct drm_i915_private *i915 = ce->engine->i915; in gen8_update_reg_state_unlocked()
1681 u32 ctx_oactxctrl = i915->perf.ctx_oactxctrl_offset; in gen8_update_reg_state_unlocked()
1682 u32 ctx_flexeu0 = i915->perf.ctx_flexeu0_offset; in gen8_update_reg_state_unlocked()
1696 (stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_update_reg_state_unlocked()
1697 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_update_reg_state_unlocked()
1707 intel_sseu_make_rpcs(i915, &ce->sseu)); in gen8_update_reg_state_unlocked()
1728 offset = i915_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE; in gen8_store_flex()
1731 *cs++ = offset + (flex->offset + 1) * sizeof(u32); in gen8_store_flex()
1733 *cs++ = flex->value; in gen8_store_flex()
1734 } while (flex++, --count); in gen8_store_flex()
1756 *cs++ = i915_mmio_reg_offset(flex->reg); in gen8_load_flex()
1757 *cs++ = flex->value; in gen8_load_flex()
1758 } while (flex++, --count); in gen8_load_flex()
1772 lockdep_assert_held(&ce->pin_mutex); in gen8_modify_context()
1774 rq = i915_request_create(ce->engine->kernel_context); in gen8_modify_context()
1811 GEM_BUG_ON(ce == ce->engine->kernel_context); in gen8_configure_context()
1813 if (ce->engine->class != RENDER_CLASS) in gen8_configure_context()
1820 flex->value = intel_sseu_make_rpcs(ctx->i915, &ce->sseu); in gen8_configure_context()
1836 * Manages updating the per-context aspects of the OA stream
1846 * won't automatically reload an out-of-date timer exponent even
1850 * - Ensure the currently running context's per-context OA state is
1852 * - Ensure that all existing contexts will have the correct per-context
1854 * - Ensure any new contexts will be initialized with the correct
1855 * per-context OA state.
1862 struct drm_i915_private *i915 = stream->dev_priv; in gen8_configure_all_contexts()
1864 const u32 ctx_flexeu0 = i915->perf.ctx_flexeu0_offset; in gen8_configure_all_contexts()
1873 i915->perf.ctx_oactxctrl_offset, in gen8_configure_all_contexts()
1874 ((stream->period_exponent << GEN8_OA_TIMER_PERIOD_SHIFT) | in gen8_configure_all_contexts()
1875 (stream->periodic ? GEN8_OA_TIMER_ENABLE : 0) | in gen8_configure_all_contexts()
1894 lockdep_assert_held(&i915->drm.struct_mutex); in gen8_configure_all_contexts()
1899 * lite-restore). This means we can't safely update a context's image, in gen8_configure_all_contexts()
1912 list_for_each_entry(ctx, &i915->contexts.list, link) { in gen8_configure_all_contexts()
1915 if (ctx == i915->kernel_context) in gen8_configure_all_contexts()
1929 struct intel_context *ce = engine->kernel_context; in gen8_configure_all_contexts()
1932 if (engine->class != RENDER_CLASS) in gen8_configure_all_contexts()
1935 regs[0].value = intel_sseu_make_rpcs(i915, &ce->sseu); in gen8_configure_all_contexts()
1947 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_enable_metric_set()
1948 const struct i915_oa_config *oa_config = stream->oa_config; in gen8_enable_metric_set()
1963 * Currently none of the high-level metrics we have depend on knowing in gen8_enable_metric_set()
1989 config_oa_regs(dev_priv, oa_config->mux_regs, oa_config->mux_regs_len); in gen8_enable_metric_set()
1992 config_oa_regs(dev_priv, oa_config->b_counter_regs, in gen8_enable_metric_set()
1993 oa_config->b_counter_regs_len); in gen8_enable_metric_set()
2000 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_disable_metric_set()
2011 struct drm_i915_private *dev_priv = stream->dev_priv; in gen10_disable_metric_set()
2023 struct drm_i915_private *dev_priv = stream->dev_priv; in gen7_oa_enable()
2024 struct i915_gem_context *ctx = stream->ctx; in gen7_oa_enable()
2025 u32 ctx_id = stream->specific_ctx_id; in gen7_oa_enable()
2026 bool periodic = stream->periodic; in gen7_oa_enable()
2027 u32 period_exponent = stream->period_exponent; in gen7_oa_enable()
2028 u32 report_format = stream->oa_buffer.format; in gen7_oa_enable()
2053 struct drm_i915_private *dev_priv = stream->dev_priv; in gen8_oa_enable()
2054 u32 report_format = stream->oa_buffer.format; in gen8_oa_enable()
2069 * filtering and instead filter on the cpu based on the context-id in gen8_oa_enable()
2078 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
2088 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_enable()
2090 dev_priv->perf.ops.oa_enable(stream); in i915_oa_stream_enable()
2092 if (stream->periodic) in i915_oa_stream_enable()
2093 hrtimer_start(&stream->poll_check_timer, in i915_oa_stream_enable()
2100 struct intel_uncore *uncore = &stream->dev_priv->uncore; in gen7_oa_disable()
2111 struct intel_uncore *uncore = &stream->dev_priv->uncore; in gen8_oa_disable()
2121 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
2130 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_disable()
2132 dev_priv->perf.ops.oa_disable(stream); in i915_oa_stream_disable()
2134 if (stream->periodic) in i915_oa_stream_disable()
2135 hrtimer_cancel(&stream->poll_check_timer); in i915_oa_stream_disable()
2148 * i915_oa_stream_init - validate combined props for OA stream and init
2169 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_oa_stream_init()
2177 if (!dev_priv->perf.metrics_kobj) { in i915_oa_stream_init()
2179 return -EINVAL; in i915_oa_stream_init()
2182 if (!(props->sample_flags & SAMPLE_OA_REPORT)) { in i915_oa_stream_init()
2184 return -EINVAL; in i915_oa_stream_init()
2187 if (!dev_priv->perf.ops.enable_metric_set) { in i915_oa_stream_init()
2189 return -ENODEV; in i915_oa_stream_init()
2196 if (dev_priv->perf.exclusive_stream) { in i915_oa_stream_init()
2198 return -EBUSY; in i915_oa_stream_init()
2201 if (!props->oa_format) { in i915_oa_stream_init()
2203 return -EINVAL; in i915_oa_stream_init()
2206 stream->sample_size = sizeof(struct drm_i915_perf_record_header); in i915_oa_stream_init()
2208 format_size = dev_priv->perf.oa_formats[props->oa_format].size; in i915_oa_stream_init()
2210 stream->sample_flags |= SAMPLE_OA_REPORT; in i915_oa_stream_init()
2211 stream->sample_size += format_size; in i915_oa_stream_init()
2213 stream->oa_buffer.format_size = format_size; in i915_oa_stream_init()
2214 if (WARN_ON(stream->oa_buffer.format_size == 0)) in i915_oa_stream_init()
2215 return -EINVAL; in i915_oa_stream_init()
2217 stream->oa_buffer.format = in i915_oa_stream_init()
2218 dev_priv->perf.oa_formats[props->oa_format].format; in i915_oa_stream_init()
2220 stream->periodic = props->oa_periodic; in i915_oa_stream_init()
2221 if (stream->periodic) in i915_oa_stream_init()
2222 stream->period_exponent = props->oa_period_exponent; in i915_oa_stream_init()
2224 if (stream->ctx) { in i915_oa_stream_init()
2232 ret = get_oa_config(dev_priv, props->metrics_set, &stream->oa_config); in i915_oa_stream_init()
2234 DRM_DEBUG("Invalid OA config id=%i\n", props->metrics_set); in i915_oa_stream_init()
2238 /* PRM - observability performance counters: in i915_oa_stream_init()
2250 stream->wakeref = intel_runtime_pm_get(&dev_priv->runtime_pm); in i915_oa_stream_init()
2251 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2257 ret = i915_mutex_lock_interruptible(&dev_priv->drm); in i915_oa_stream_init()
2261 stream->ops = &i915_oa_stream_ops; in i915_oa_stream_init()
2262 dev_priv->perf.exclusive_stream = stream; in i915_oa_stream_init()
2264 ret = dev_priv->perf.ops.enable_metric_set(stream); in i915_oa_stream_init()
2270 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_init()
2272 hrtimer_init(&stream->poll_check_timer, in i915_oa_stream_init()
2274 stream->poll_check_timer.function = oa_poll_check_timer_cb; in i915_oa_stream_init()
2275 init_waitqueue_head(&stream->poll_wq); in i915_oa_stream_init()
2276 spin_lock_init(&stream->oa_buffer.ptr_lock); in i915_oa_stream_init()
2281 dev_priv->perf.exclusive_stream = NULL; in i915_oa_stream_init()
2282 dev_priv->perf.ops.disable_metric_set(stream); in i915_oa_stream_init()
2283 mutex_unlock(&dev_priv->drm.struct_mutex); in i915_oa_stream_init()
2289 put_oa_config(dev_priv, stream->oa_config); in i915_oa_stream_init()
2291 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL); in i915_oa_stream_init()
2292 intel_runtime_pm_put(&dev_priv->runtime_pm, stream->wakeref); in i915_oa_stream_init()
2295 if (stream->ctx) in i915_oa_stream_init()
2307 if (engine->class != RENDER_CLASS) in i915_oa_init_reg_state()
2310 stream = engine->i915->perf.exclusive_stream; in i915_oa_init_reg_state()
2312 gen8_update_reg_state_unlocked(stream, ce, regs, stream->oa_config); in i915_oa_init_reg_state()
2316 * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation
2323 * Besides wrapping &i915_perf_stream_ops->read this provides a common place to
2327 * For example ret will be -ENOSPC whenever there is more buffered data than
2332 * Another case with ret == -EFAULT is more of a grey area since it would seem
2350 * stream->ops->read() implementations. in i915_perf_read_locked()
2353 int ret = stream->ops->read(stream, buf, count, &offset); in i915_perf_read_locked()
2355 return offset ?: (ret ?: -EAGAIN); in i915_perf_read_locked()
2359 * i915_perf_read - handles read() FOP for i915 perf stream FDs
2367 * &i915_perf_stream_ops->read but to save having stream implementations (of
2381 struct i915_perf_stream *stream = file->private_data; in i915_perf_read()
2382 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_read()
2389 if (!stream->enabled) in i915_perf_read()
2390 return -EIO; in i915_perf_read()
2392 if (!(file->f_flags & O_NONBLOCK)) { in i915_perf_read()
2394 * stream->ops->wait_unlocked. in i915_perf_read()
2401 ret = stream->ops->wait_unlocked(stream); in i915_perf_read()
2405 mutex_lock(&dev_priv->perf.lock); in i915_perf_read()
2408 mutex_unlock(&dev_priv->perf.lock); in i915_perf_read()
2409 } while (ret == -EAGAIN); in i915_perf_read()
2411 mutex_lock(&dev_priv->perf.lock); in i915_perf_read()
2413 mutex_unlock(&dev_priv->perf.lock); in i915_perf_read()
2420 * and read() returning -EAGAIN. Clearing the oa.pollin state here in i915_perf_read()
2424 if (ret >= 0 || ret == -EAGAIN) { in i915_perf_read()
2425 /* Maybe make ->pollin per-stream state if we support multiple in i915_perf_read()
2428 stream->pollin = false; in i915_perf_read()
2440 stream->pollin = true; in oa_poll_check_timer_cb()
2441 wake_up(&stream->poll_wq); in oa_poll_check_timer_cb()
2450 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
2457 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
2460 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2461 * with any non-file-operation driver hooks.
2472 stream->ops->poll_wait(stream, file, wait); in i915_perf_poll_locked()
2480 if (stream->pollin) in i915_perf_poll_locked()
2487 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
2501 struct i915_perf_stream *stream = file->private_data; in i915_perf_poll()
2502 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_poll()
2505 mutex_lock(&dev_priv->perf.lock); in i915_perf_poll()
2507 mutex_unlock(&dev_priv->perf.lock); in i915_perf_poll()
2513 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
2524 if (stream->enabled) in i915_perf_enable_locked()
2527 /* Allow stream->ops->enable() to refer to this */ in i915_perf_enable_locked()
2528 stream->enabled = true; in i915_perf_enable_locked()
2530 if (stream->ops->enable) in i915_perf_enable_locked()
2531 stream->ops->enable(stream); in i915_perf_enable_locked()
2535 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
2540 * The intention is that disabling an re-enabling a stream will ideally be
2541 * cheaper than destroying and re-opening a stream with the same configuration,
2543 * must be retained between disabling and re-enabling a stream.
2546 * to attempt to read from the stream (-EIO).
2550 if (!stream->enabled) in i915_perf_disable_locked()
2553 /* Allow stream->ops->disable() to refer to this */ in i915_perf_disable_locked()
2554 stream->enabled = false; in i915_perf_disable_locked()
2556 if (stream->ops->disable) in i915_perf_disable_locked()
2557 stream->ops->disable(stream); in i915_perf_disable_locked()
2561 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
2566 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2567 * with any non-file-operation driver hooks.
2569 * Returns: zero on success or a negative error code. Returns -EINVAL for
2585 return -EINVAL; in i915_perf_ioctl_locked()
2589 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
2596 * Returns: zero on success or a negative error code. Returns -EINVAL for
2603 struct i915_perf_stream *stream = file->private_data; in i915_perf_ioctl()
2604 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_ioctl()
2607 mutex_lock(&dev_priv->perf.lock); in i915_perf_ioctl()
2609 mutex_unlock(&dev_priv->perf.lock); in i915_perf_ioctl()
2615 * i915_perf_destroy_locked - destroy an i915 perf stream
2621 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2622 * with any non-file-operation driver hooks.
2626 if (stream->enabled) in i915_perf_destroy_locked()
2629 if (stream->ops->destroy) in i915_perf_destroy_locked()
2630 stream->ops->destroy(stream); in i915_perf_destroy_locked()
2632 list_del(&stream->link); in i915_perf_destroy_locked()
2634 if (stream->ctx) in i915_perf_destroy_locked()
2635 i915_gem_context_put(stream->ctx); in i915_perf_destroy_locked()
2641 * i915_perf_release - handles userspace close() of a stream file
2653 struct i915_perf_stream *stream = file->private_data; in i915_perf_release()
2654 struct drm_i915_private *dev_priv = stream->dev_priv; in i915_perf_release()
2656 mutex_lock(&dev_priv->perf.lock); in i915_perf_release()
2658 mutex_unlock(&dev_priv->perf.lock); in i915_perf_release()
2661 drm_dev_put(&dev_priv->drm); in i915_perf_release()
2682 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
2691 * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex
2692 * taken to serialize with any non-file-operation driver hooks.
2718 if (props->single_context) { in i915_perf_open_ioctl_locked()
2719 u32 ctx_handle = props->ctx_handle; in i915_perf_open_ioctl_locked()
2720 struct drm_i915_file_private *file_priv = file->driver_priv; in i915_perf_open_ioctl_locked()
2726 ret = -ENOENT; in i915_perf_open_ioctl_locked()
2735 * non-privileged client. in i915_perf_open_ioctl_locked()
2739 * from updating as system-wide / global values. Even though we can in i915_perf_open_ioctl_locked()
2755 DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n"); in i915_perf_open_ioctl_locked()
2756 ret = -EACCES; in i915_perf_open_ioctl_locked()
2762 ret = -ENOMEM; in i915_perf_open_ioctl_locked()
2766 stream->dev_priv = dev_priv; in i915_perf_open_ioctl_locked()
2767 stream->ctx = specific_ctx; in i915_perf_open_ioctl_locked()
2773 /* we avoid simply assigning stream->sample_flags = props->sample_flags in i915_perf_open_ioctl_locked()
2777 if (WARN_ON(stream->sample_flags != props->sample_flags)) { in i915_perf_open_ioctl_locked()
2778 ret = -ENODEV; in i915_perf_open_ioctl_locked()
2782 list_add(&stream->link, &dev_priv->perf.streams); in i915_perf_open_ioctl_locked()
2784 if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) in i915_perf_open_ioctl_locked()
2786 if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) in i915_perf_open_ioctl_locked()
2795 if (!(param->flags & I915_PERF_FLAG_DISABLED)) in i915_perf_open_ioctl_locked()
2801 drm_dev_get(&dev_priv->drm); in i915_perf_open_ioctl_locked()
2806 list_del(&stream->link); in i915_perf_open_ioctl_locked()
2808 if (stream->ops->destroy) in i915_perf_open_ioctl_locked()
2809 stream->ops->destroy(stream); in i915_perf_open_ioctl_locked()
2822 1000ULL * RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz); in oa_exponent_to_ns()
2826 * read_properties_unlocked - validate + copy userspace stream open properties
2852 return -EINVAL; in read_properties_unlocked()
2863 return -EINVAL; in read_properties_unlocked()
2881 return -EINVAL; in read_properties_unlocked()
2886 props->single_context = 1; in read_properties_unlocked()
2887 props->ctx_handle = value; in read_properties_unlocked()
2891 props->sample_flags |= SAMPLE_OA_REPORT; in read_properties_unlocked()
2896 return -EINVAL; in read_properties_unlocked()
2898 props->metrics_set = value; in read_properties_unlocked()
2902 DRM_DEBUG("Out-of-range OA report format %llu\n", in read_properties_unlocked()
2904 return -EINVAL; in read_properties_unlocked()
2906 if (!dev_priv->perf.oa_formats[value].size) { in read_properties_unlocked()
2909 return -EINVAL; in read_properties_unlocked()
2911 props->oa_format = value; in read_properties_unlocked()
2917 return -EINVAL; in read_properties_unlocked()
2946 return -EACCES; in read_properties_unlocked()
2949 props->oa_periodic = true; in read_properties_unlocked()
2950 props->oa_period_exponent = value; in read_properties_unlocked()
2954 return -EINVAL; in read_properties_unlocked()
2964 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
2974 * i915-perf stream is expected to be a suitable interface for other forms of
2981 * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock
2982 * mutex for serializing with any non-file-operation driver hooks.
2990 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_open_ioctl()
2996 if (!dev_priv->perf.initialized) { in i915_perf_open_ioctl()
2998 return -ENOTSUPP; in i915_perf_open_ioctl()
3004 if (param->flags & ~known_open_flags) { in i915_perf_open_ioctl()
3006 return -EINVAL; in i915_perf_open_ioctl()
3010 u64_to_user_ptr(param->properties_ptr), in i915_perf_open_ioctl()
3011 param->num_properties, in i915_perf_open_ioctl()
3016 mutex_lock(&dev_priv->perf.lock); in i915_perf_open_ioctl()
3018 mutex_unlock(&dev_priv->perf.lock); in i915_perf_open_ioctl()
3024 * i915_perf_register - exposes i915-perf to userspace
3029 * used to open an i915-perf stream.
3035 if (!dev_priv->perf.initialized) in i915_perf_register()
3042 mutex_lock(&dev_priv->perf.lock); in i915_perf_register()
3044 dev_priv->perf.metrics_kobj = in i915_perf_register()
3046 &dev_priv->drm.primary->kdev->kobj); in i915_perf_register()
3047 if (!dev_priv->perf.metrics_kobj) in i915_perf_register()
3050 sysfs_attr_init(&dev_priv->perf.test_config.sysfs_metric_id.attr); in i915_perf_register()
3085 if (dev_priv->perf.test_config.id == 0) in i915_perf_register()
3088 ret = sysfs_create_group(dev_priv->perf.metrics_kobj, in i915_perf_register()
3089 &dev_priv->perf.test_config.sysfs_metric); in i915_perf_register()
3093 atomic_set(&dev_priv->perf.test_config.ref_count, 1); in i915_perf_register()
3098 kobject_put(dev_priv->perf.metrics_kobj); in i915_perf_register()
3099 dev_priv->perf.metrics_kobj = NULL; in i915_perf_register()
3102 mutex_unlock(&dev_priv->perf.lock); in i915_perf_register()
3106 * i915_perf_unregister - hide i915-perf from userspace
3109 * i915-perf state cleanup is split up into an 'unregister' and
3116 if (!dev_priv->perf.metrics_kobj) in i915_perf_unregister()
3119 sysfs_remove_group(dev_priv->perf.metrics_kobj, in i915_perf_unregister()
3120 &dev_priv->perf.test_config.sysfs_metric); in i915_perf_unregister()
3122 kobject_put(dev_priv->perf.metrics_kobj); in i915_perf_unregister()
3123 dev_priv->perf.metrics_kobj = NULL; in i915_perf_unregister()
3230 return ERR_PTR(-EFAULT); in alloc_oa_regs()
3235 return ERR_PTR(-EINVAL); in alloc_oa_regs()
3239 return ERR_PTR(-ENOMEM); in alloc_oa_regs()
3250 err = -EINVAL; in alloc_oa_regs()
3278 return sprintf(buf, "%d\n", oa_config->id); in show_dynamic_id()
3284 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); in create_dynamic_oa_sysfs_entry()
3285 oa_config->sysfs_metric_id.attr.name = "id"; in create_dynamic_oa_sysfs_entry()
3286 oa_config->sysfs_metric_id.attr.mode = S_IRUGO; in create_dynamic_oa_sysfs_entry()
3287 oa_config->sysfs_metric_id.show = show_dynamic_id; in create_dynamic_oa_sysfs_entry()
3288 oa_config->sysfs_metric_id.store = NULL; in create_dynamic_oa_sysfs_entry()
3290 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; in create_dynamic_oa_sysfs_entry()
3291 oa_config->attrs[1] = NULL; in create_dynamic_oa_sysfs_entry()
3293 oa_config->sysfs_metric.name = oa_config->uuid; in create_dynamic_oa_sysfs_entry()
3294 oa_config->sysfs_metric.attrs = oa_config->attrs; in create_dynamic_oa_sysfs_entry()
3296 return sysfs_create_group(dev_priv->perf.metrics_kobj, in create_dynamic_oa_sysfs_entry()
3297 &oa_config->sysfs_metric); in create_dynamic_oa_sysfs_entry()
3301 * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
3316 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_add_config_ioctl()
3321 if (!dev_priv->perf.initialized) { in i915_perf_add_config_ioctl()
3323 return -ENOTSUPP; in i915_perf_add_config_ioctl()
3326 if (!dev_priv->perf.metrics_kobj) { in i915_perf_add_config_ioctl()
3328 return -EINVAL; in i915_perf_add_config_ioctl()
3333 return -EACCES; in i915_perf_add_config_ioctl()
3336 if ((!args->mux_regs_ptr || !args->n_mux_regs) && in i915_perf_add_config_ioctl()
3337 (!args->boolean_regs_ptr || !args->n_boolean_regs) && in i915_perf_add_config_ioctl()
3338 (!args->flex_regs_ptr || !args->n_flex_regs)) { in i915_perf_add_config_ioctl()
3340 return -EINVAL; in i915_perf_add_config_ioctl()
3346 return -ENOMEM; in i915_perf_add_config_ioctl()
3349 atomic_set(&oa_config->ref_count, 1); in i915_perf_add_config_ioctl()
3351 if (!uuid_is_valid(args->uuid)) { in i915_perf_add_config_ioctl()
3353 err = -EINVAL; in i915_perf_add_config_ioctl()
3357 /* Last character in oa_config->uuid will be 0 because oa_config is in i915_perf_add_config_ioctl()
3360 memcpy(oa_config->uuid, args->uuid, sizeof(args->uuid)); in i915_perf_add_config_ioctl()
3362 oa_config->mux_regs_len = args->n_mux_regs; in i915_perf_add_config_ioctl()
3363 oa_config->mux_regs = in i915_perf_add_config_ioctl()
3365 dev_priv->perf.ops.is_valid_mux_reg, in i915_perf_add_config_ioctl()
3366 u64_to_user_ptr(args->mux_regs_ptr), in i915_perf_add_config_ioctl()
3367 args->n_mux_regs); in i915_perf_add_config_ioctl()
3369 if (IS_ERR(oa_config->mux_regs)) { in i915_perf_add_config_ioctl()
3371 err = PTR_ERR(oa_config->mux_regs); in i915_perf_add_config_ioctl()
3375 oa_config->b_counter_regs_len = args->n_boolean_regs; in i915_perf_add_config_ioctl()
3376 oa_config->b_counter_regs = in i915_perf_add_config_ioctl()
3378 dev_priv->perf.ops.is_valid_b_counter_reg, in i915_perf_add_config_ioctl()
3379 u64_to_user_ptr(args->boolean_regs_ptr), in i915_perf_add_config_ioctl()
3380 args->n_boolean_regs); in i915_perf_add_config_ioctl()
3382 if (IS_ERR(oa_config->b_counter_regs)) { in i915_perf_add_config_ioctl()
3384 err = PTR_ERR(oa_config->b_counter_regs); in i915_perf_add_config_ioctl()
3389 if (args->n_flex_regs != 0) { in i915_perf_add_config_ioctl()
3390 err = -EINVAL; in i915_perf_add_config_ioctl()
3394 oa_config->flex_regs_len = args->n_flex_regs; in i915_perf_add_config_ioctl()
3395 oa_config->flex_regs = in i915_perf_add_config_ioctl()
3397 dev_priv->perf.ops.is_valid_flex_reg, in i915_perf_add_config_ioctl()
3398 u64_to_user_ptr(args->flex_regs_ptr), in i915_perf_add_config_ioctl()
3399 args->n_flex_regs); in i915_perf_add_config_ioctl()
3401 if (IS_ERR(oa_config->flex_regs)) { in i915_perf_add_config_ioctl()
3403 err = PTR_ERR(oa_config->flex_regs); in i915_perf_add_config_ioctl()
3408 err = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3415 idr_for_each_entry(&dev_priv->perf.metrics_idr, tmp, id) { in i915_perf_add_config_ioctl()
3416 if (!strcmp(tmp->uuid, oa_config->uuid)) { in i915_perf_add_config_ioctl()
3418 err = -EADDRINUSE; in i915_perf_add_config_ioctl()
3430 oa_config->id = idr_alloc(&dev_priv->perf.metrics_idr, in i915_perf_add_config_ioctl()
3433 if (oa_config->id < 0) { in i915_perf_add_config_ioctl()
3435 err = oa_config->id; in i915_perf_add_config_ioctl()
3439 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3441 DRM_DEBUG("Added config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_add_config_ioctl()
3443 return oa_config->id; in i915_perf_add_config_ioctl()
3446 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_add_config_ioctl()
3454 * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
3467 struct drm_i915_private *dev_priv = dev->dev_private; in i915_perf_remove_config_ioctl()
3472 if (!dev_priv->perf.initialized) { in i915_perf_remove_config_ioctl()
3474 return -ENOTSUPP; in i915_perf_remove_config_ioctl()
3479 return -EACCES; in i915_perf_remove_config_ioctl()
3482 ret = mutex_lock_interruptible(&dev_priv->perf.metrics_lock); in i915_perf_remove_config_ioctl()
3486 oa_config = idr_find(&dev_priv->perf.metrics_idr, *arg); in i915_perf_remove_config_ioctl()
3489 ret = -ENOENT; in i915_perf_remove_config_ioctl()
3493 GEM_BUG_ON(*arg != oa_config->id); in i915_perf_remove_config_ioctl()
3495 sysfs_remove_group(dev_priv->perf.metrics_kobj, in i915_perf_remove_config_ioctl()
3496 &oa_config->sysfs_metric); in i915_perf_remove_config_ioctl()
3498 idr_remove(&dev_priv->perf.metrics_idr, *arg); in i915_perf_remove_config_ioctl()
3500 DRM_DEBUG("Removed config %s id=%i\n", oa_config->uuid, oa_config->id); in i915_perf_remove_config_ioctl()
3505 mutex_unlock(&dev_priv->perf.metrics_lock); in i915_perf_remove_config_ioctl()
3553 * i915_perf_init - initialize i915-perf state on module load
3556 * Initializes i915-perf state without exposing anything to userspace.
3558 * Note: i915-perf initialization is split into an 'init' and 'register'
3564 dev_priv->perf.ops.is_valid_b_counter_reg = in i915_perf_init()
3566 dev_priv->perf.ops.is_valid_mux_reg = in i915_perf_init()
3568 dev_priv->perf.ops.is_valid_flex_reg = NULL; in i915_perf_init()
3569 dev_priv->perf.ops.enable_metric_set = hsw_enable_metric_set; in i915_perf_init()
3570 dev_priv->perf.ops.disable_metric_set = hsw_disable_metric_set; in i915_perf_init()
3571 dev_priv->perf.ops.oa_enable = gen7_oa_enable; in i915_perf_init()
3572 dev_priv->perf.ops.oa_disable = gen7_oa_disable; in i915_perf_init()
3573 dev_priv->perf.ops.read = gen7_oa_read; in i915_perf_init()
3574 dev_priv->perf.ops.oa_hw_tail_read = in i915_perf_init()
3577 dev_priv->perf.oa_formats = hsw_oa_formats; in i915_perf_init()
3585 dev_priv->perf.oa_formats = gen8_plus_oa_formats; in i915_perf_init()
3587 dev_priv->perf.ops.oa_enable = gen8_oa_enable; in i915_perf_init()
3588 dev_priv->perf.ops.oa_disable = gen8_oa_disable; in i915_perf_init()
3589 dev_priv->perf.ops.read = gen8_oa_read; in i915_perf_init()
3590 dev_priv->perf.ops.oa_hw_tail_read = gen8_oa_hw_tail_read; in i915_perf_init()
3593 dev_priv->perf.ops.is_valid_b_counter_reg = in i915_perf_init()
3595 dev_priv->perf.ops.is_valid_mux_reg = in i915_perf_init()
3597 dev_priv->perf.ops.is_valid_flex_reg = in i915_perf_init()
3601 dev_priv->perf.ops.is_valid_mux_reg = in i915_perf_init()
3605 dev_priv->perf.ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
3606 dev_priv->perf.ops.disable_metric_set = gen8_disable_metric_set; in i915_perf_init()
3609 dev_priv->perf.ctx_oactxctrl_offset = 0x120; in i915_perf_init()
3610 dev_priv->perf.ctx_flexeu0_offset = 0x2ce; in i915_perf_init()
3612 dev_priv->perf.gen8_valid_ctx_bit = BIT(25); in i915_perf_init()
3614 dev_priv->perf.ctx_oactxctrl_offset = 0x128; in i915_perf_init()
3615 dev_priv->perf.ctx_flexeu0_offset = 0x3de; in i915_perf_init()
3617 dev_priv->perf.gen8_valid_ctx_bit = BIT(16); in i915_perf_init()
3620 dev_priv->perf.ops.is_valid_b_counter_reg = in i915_perf_init()
3622 dev_priv->perf.ops.is_valid_mux_reg = in i915_perf_init()
3624 dev_priv->perf.ops.is_valid_flex_reg = in i915_perf_init()
3627 dev_priv->perf.ops.enable_metric_set = gen8_enable_metric_set; in i915_perf_init()
3628 dev_priv->perf.ops.disable_metric_set = gen10_disable_metric_set; in i915_perf_init()
3631 dev_priv->perf.ctx_oactxctrl_offset = 0x128; in i915_perf_init()
3632 dev_priv->perf.ctx_flexeu0_offset = 0x3de; in i915_perf_init()
3634 dev_priv->perf.ctx_oactxctrl_offset = 0x124; in i915_perf_init()
3635 dev_priv->perf.ctx_flexeu0_offset = 0x78e; in i915_perf_init()
3637 dev_priv->perf.gen8_valid_ctx_bit = BIT(16); in i915_perf_init()
3641 if (dev_priv->perf.ops.enable_metric_set) { in i915_perf_init()
3642 INIT_LIST_HEAD(&dev_priv->perf.streams); in i915_perf_init()
3643 mutex_init(&dev_priv->perf.lock); in i915_perf_init()
3646 (RUNTIME_INFO(dev_priv)->cs_timestamp_frequency_khz / 2); in i915_perf_init()
3647 dev_priv->perf.sysctl_header = register_sysctl_table(dev_root); in i915_perf_init()
3649 mutex_init(&dev_priv->perf.metrics_lock); in i915_perf_init()
3650 idr_init(&dev_priv->perf.metrics_idr); in i915_perf_init()
3662 ratelimit_state_init(&dev_priv->perf.spurious_report_rs, in i915_perf_init()
3668 ratelimit_set_flags(&dev_priv->perf.spurious_report_rs, in i915_perf_init()
3671 dev_priv->perf.initialized = true; in i915_perf_init()
3686 * i915_perf_fini - Counter part to i915_perf_init()
3691 if (!dev_priv->perf.initialized) in i915_perf_fini()
3694 idr_for_each(&dev_priv->perf.metrics_idr, destroy_config, dev_priv); in i915_perf_fini()
3695 idr_destroy(&dev_priv->perf.metrics_idr); in i915_perf_fini()
3697 unregister_sysctl_table(dev_priv->perf.sysctl_header); in i915_perf_fini()
3699 memset(&dev_priv->perf.ops, 0, sizeof(dev_priv->perf.ops)); in i915_perf_fini()
3701 dev_priv->perf.initialized = false; in i915_perf_fini()