Lines Matching refs:pt
138 struct intel_pt *pt; member
179 static void intel_pt_dump(struct intel_pt *pt __maybe_unused, in intel_pt_dump()
219 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf, in intel_pt_dump_event() argument
223 intel_pt_dump(pt, buf, len); in intel_pt_dump_event()
236 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a, in intel_pt_do_fix_overlap() argument
243 pt->have_tsc, &consecutive); in intel_pt_do_fix_overlap()
261 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_buffer()
268 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_buffer()
270 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_buffer()
312 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_lookahead()
362 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
507 static inline u8 intel_pt_cpumode(struct intel_pt *pt, uint64_t ip) in intel_pt_cpumode() argument
509 return ip >= pt->kernel_start ? in intel_pt_cpumode()
520 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
536 cpumode = intel_pt_cpumode(ptq->pt, *ip); in intel_pt_walk_next_insn()
542 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
644 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip, in intel_pt_match_pgd_ip() argument
652 list_for_each_entry(filt, &pt->filts.head, list) { in intel_pt_match_pgd_ip()
690 if (ip >= ptq->pt->kernel_start) in __intel_pt_pgd_ip()
691 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
704 return intel_pt_match_pgd_ip(ptq->pt, ip, offset, in __intel_pt_pgd_ip()
713 static bool intel_pt_get_config(struct intel_pt *pt, in intel_pt_get_config() argument
716 if (attr->type == pt->pmu_type) { in intel_pt_get_config()
725 static bool intel_pt_exclude_kernel(struct intel_pt *pt) in intel_pt_exclude_kernel() argument
729 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_exclude_kernel()
730 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) && in intel_pt_exclude_kernel()
737 static bool intel_pt_return_compression(struct intel_pt *pt) in intel_pt_return_compression() argument
742 if (!pt->noretcomp_bit) in intel_pt_return_compression()
745 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_return_compression()
746 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_return_compression()
747 (config & pt->noretcomp_bit)) in intel_pt_return_compression()
753 static bool intel_pt_branch_enable(struct intel_pt *pt) in intel_pt_branch_enable() argument
758 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_branch_enable()
759 if (intel_pt_get_config(pt, &evsel->core.attr, &config) && in intel_pt_branch_enable()
766 static unsigned int intel_pt_mtc_period(struct intel_pt *pt) in intel_pt_mtc_period() argument
772 if (!pt->mtc_freq_bits) in intel_pt_mtc_period()
775 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++) in intel_pt_mtc_period()
778 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_mtc_period()
779 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) in intel_pt_mtc_period()
780 return (config & pt->mtc_freq_bits) >> shift; in intel_pt_mtc_period()
785 static bool intel_pt_timeless_decoding(struct intel_pt *pt) in intel_pt_timeless_decoding() argument
791 if (!pt->tsc_bit || !pt->cap_user_time_zero) in intel_pt_timeless_decoding()
794 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_timeless_decoding()
797 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) { in intel_pt_timeless_decoding()
798 if (config & pt->tsc_bit) in intel_pt_timeless_decoding()
807 static bool intel_pt_tracing_kernel(struct intel_pt *pt) in intel_pt_tracing_kernel() argument
811 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_tracing_kernel()
812 if (intel_pt_get_config(pt, &evsel->core.attr, NULL) && in intel_pt_tracing_kernel()
819 static bool intel_pt_have_tsc(struct intel_pt *pt) in intel_pt_have_tsc() argument
825 if (!pt->tsc_bit) in intel_pt_have_tsc()
828 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_have_tsc()
829 if (intel_pt_get_config(pt, &evsel->core.attr, &config)) { in intel_pt_have_tsc()
830 if (config & pt->tsc_bit) in intel_pt_have_tsc()
839 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns) in intel_pt_ns_to_ticks() argument
843 quot = ns / pt->tc.time_mult; in intel_pt_ns_to_ticks()
844 rem = ns % pt->tc.time_mult; in intel_pt_ns_to_ticks()
845 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) / in intel_pt_ns_to_ticks()
846 pt->tc.time_mult; in intel_pt_ns_to_ticks()
849 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, in intel_pt_alloc_queue() argument
853 struct perf_env *env = pt->machine->env; in intel_pt_alloc_queue()
860 if (pt->synth_opts.callchain) { in intel_pt_alloc_queue()
864 sz += (pt->synth_opts.callchain_sz + 1) * sizeof(u64); in intel_pt_alloc_queue()
870 if (pt->synth_opts.last_branch) { in intel_pt_alloc_queue()
873 sz += pt->synth_opts.last_branch_sz * in intel_pt_alloc_queue()
887 ptq->pt = pt; in intel_pt_alloc_queue()
889 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
899 params.return_compression = intel_pt_return_compression(pt); in intel_pt_alloc_queue()
900 params.branch_enable = intel_pt_branch_enable(pt); in intel_pt_alloc_queue()
901 params.max_non_turbo_ratio = pt->max_non_turbo_ratio; in intel_pt_alloc_queue()
902 params.mtc_period = intel_pt_mtc_period(pt); in intel_pt_alloc_queue()
903 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n; in intel_pt_alloc_queue()
904 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d; in intel_pt_alloc_queue()
906 if (pt->filts.cnt > 0) in intel_pt_alloc_queue()
909 if (pt->synth_opts.instructions) { in intel_pt_alloc_queue()
910 if (pt->synth_opts.period) { in intel_pt_alloc_queue()
911 switch (pt->synth_opts.period_type) { in intel_pt_alloc_queue()
915 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
919 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
923 params.period = intel_pt_ns_to_ticks(pt, in intel_pt_alloc_queue()
924 pt->synth_opts.period); in intel_pt_alloc_queue()
970 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt, in intel_pt_set_pid_tid_cpu() argument
975 if (queue->tid == -1 || pt->have_sched_switch) { in intel_pt_set_pid_tid_cpu()
976 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
981 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
1021 static void intel_pt_setup_time_range(struct intel_pt *pt, in intel_pt_setup_time_range() argument
1024 if (!pt->range_cnt) in intel_pt_setup_time_range()
1027 ptq->sel_timestamp = pt->time_ranges[0].start; in intel_pt_setup_time_range()
1033 ptq->sel_timestamp = pt->time_ranges[0].end; in intel_pt_setup_time_range()
1038 static int intel_pt_setup_queue(struct intel_pt *pt, in intel_pt_setup_queue() argument
1048 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
1059 if (pt->sampling_mode && !pt->snapshot_mode && in intel_pt_setup_queue()
1060 pt->timeless_decoding) in intel_pt_setup_queue()
1063 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
1065 intel_pt_setup_time_range(pt, ptq); in intel_pt_setup_queue()
1074 if (pt->timeless_decoding) in intel_pt_setup_queue()
1111 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1120 static int intel_pt_setup_queues(struct intel_pt *pt) in intel_pt_setup_queues() argument
1125 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_setup_queues()
1126 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i); in intel_pt_setup_queues()
1144 nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos; in intel_pt_copy_last_branch_rb()
1149 if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) { in intel_pt_copy_last_branch_rb()
1169 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz; in intel_pt_update_last_branch_rb()
1179 be->flags.mispred = ptq->pt->mispred_all; in intel_pt_update_last_branch_rb()
1181 if (bs->nr < ptq->pt->synth_opts.last_branch_sz) in intel_pt_update_last_branch_rb()
1185 static inline bool intel_pt_skip_event(struct intel_pt *pt) in intel_pt_skip_event() argument
1187 return pt->synth_opts.initial_skip && in intel_pt_skip_event()
1188 pt->num_events++ < pt->synth_opts.initial_skip; in intel_pt_skip_event()
1196 static inline bool intel_pt_skip_cbr_event(struct intel_pt *pt) in intel_pt_skip_cbr_event() argument
1198 return pt->synth_opts.initial_skip && in intel_pt_skip_cbr_event()
1199 pt->num_events + 4 < pt->synth_opts.initial_skip; in intel_pt_skip_cbr_event()
1216 static void intel_pt_prep_b_sample(struct intel_pt *pt, in intel_pt_prep_b_sample() argument
1223 if (!pt->timeless_decoding) in intel_pt_prep_b_sample()
1224 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1227 sample->cpumode = intel_pt_cpumode(pt, sample->ip); in intel_pt_prep_b_sample()
1242 static inline int intel_pt_opt_inject(struct intel_pt *pt, in intel_pt_opt_inject() argument
1246 if (!pt->synth_opts.inject) in intel_pt_opt_inject()
1252 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt, in intel_pt_deliver_synth_b_event() argument
1258 ret = intel_pt_opt_inject(pt, event, sample, type); in intel_pt_deliver_synth_b_event()
1262 ret = perf_session__deliver_synth_event(pt->session, event, sample); in intel_pt_deliver_synth_b_event()
1271 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample() local
1279 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1282 if (intel_pt_skip_event(pt)) in intel_pt_synth_branch_sample()
1285 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1287 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1288 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1294 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) { in intel_pt_synth_branch_sample()
1312 return intel_pt_deliver_synth_b_event(pt, event, &sample, in intel_pt_synth_branch_sample()
1313 pt->branches_sample_type); in intel_pt_synth_branch_sample()
1316 static void intel_pt_prep_sample(struct intel_pt *pt, in intel_pt_prep_sample() argument
1321 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1323 if (pt->synth_opts.callchain) { in intel_pt_prep_sample()
1325 pt->synth_opts.callchain_sz + 1, in intel_pt_prep_sample()
1326 sample->ip, pt->kernel_start); in intel_pt_prep_sample()
1330 if (pt->synth_opts.last_branch) { in intel_pt_prep_sample()
1336 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt, in intel_pt_deliver_synth_event() argument
1344 ret = intel_pt_deliver_synth_b_event(pt, event, sample, type); in intel_pt_deliver_synth_event()
1346 if (pt->synth_opts.last_branch) in intel_pt_deliver_synth_event()
1354 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample() local
1358 if (intel_pt_skip_event(pt)) in intel_pt_synth_instruction_sample()
1361 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1363 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1364 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1376 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_instruction_sample()
1377 pt->instructions_sample_type); in intel_pt_synth_instruction_sample()
1382 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample() local
1386 if (intel_pt_skip_event(pt)) in intel_pt_synth_transaction_sample()
1389 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1391 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1392 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1394 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_transaction_sample()
1395 pt->transactions_sample_type); in intel_pt_synth_transaction_sample()
1398 static void intel_pt_prep_p_sample(struct intel_pt *pt, in intel_pt_prep_p_sample() argument
1403 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1415 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample() local
1420 if (intel_pt_skip_event(pt)) in intel_pt_synth_ptwrite_sample()
1423 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1425 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1426 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1435 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_ptwrite_sample()
1436 pt->ptwrites_sample_type); in intel_pt_synth_ptwrite_sample()
1441 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample() local
1447 if (intel_pt_skip_cbr_event(pt)) in intel_pt_synth_cbr_sample()
1452 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1454 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1455 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1457 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1459 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz); in intel_pt_synth_cbr_sample()
1465 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_cbr_sample()
1466 pt->pwr_events_sample_type); in intel_pt_synth_cbr_sample()
1471 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample() local
1476 if (intel_pt_skip_event(pt)) in intel_pt_synth_mwait_sample()
1479 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
1481 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1482 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1490 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_mwait_sample()
1491 pt->pwr_events_sample_type); in intel_pt_synth_mwait_sample()
1496 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample() local
1501 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwre_sample()
1504 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
1506 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1507 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1515 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_pwre_sample()
1516 pt->pwr_events_sample_type); in intel_pt_synth_pwre_sample()
1521 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample() local
1526 if (intel_pt_skip_event(pt)) in intel_pt_synth_exstop_sample()
1529 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
1531 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1532 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1540 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_exstop_sample()
1541 pt->pwr_events_sample_type); in intel_pt_synth_exstop_sample()
1546 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample() local
1551 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwrx_sample()
1554 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
1556 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1557 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1565 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_pwrx_sample()
1566 pt->pwr_events_sample_type); in intel_pt_synth_pwrx_sample()
1705 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pebs_sample() local
1706 struct evsel *evsel = pt->pebs_evsel; in intel_pt_synth_pebs_sample()
1711 if (intel_pt_skip_event(pt)) in intel_pt_synth_pebs_sample()
1731 cpumode = sample.ip < ptq->pt->kernel_start ? in intel_pt_synth_pebs_sample()
1744 else if (!pt->timeless_decoding) in intel_pt_synth_pebs_sample()
1747 sample.time = tsc_to_perf_time(timestamp, &pt->tc); in intel_pt_synth_pebs_sample()
1751 pt->synth_opts.callchain) { in intel_pt_synth_pebs_sample()
1753 pt->synth_opts.callchain_sz, sample.ip, in intel_pt_synth_pebs_sample()
1754 pt->kernel_start); in intel_pt_synth_pebs_sample()
1785 } else if (pt->synth_opts.last_branch) { in intel_pt_synth_pebs_sample()
1821 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, sample_type); in intel_pt_synth_pebs_sample()
1824 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu, in intel_pt_synth_error() argument
1836 err = perf_session__deliver_synth_event(pt->session, &event, NULL); in intel_pt_synth_error()
1847 struct intel_pt *pt = ptq->pt; in intel_ptq_synth_error() local
1850 tm = pt->timeless_decoding ? 0 : tsc_to_perf_time(tm, &pt->tc); in intel_ptq_synth_error()
1852 return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid, in intel_ptq_synth_error()
1856 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
1867 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
1869 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
1870 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_next_tid()
1879 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip() local
1881 return ip == pt->switch_ip && in intel_pt_is_switch_ip()
1893 struct intel_pt *pt = ptq->pt; in intel_pt_sample() local
1914 if (pt->sample_pebs && state->type & INTEL_PT_BLK_ITEMS) { in intel_pt_sample()
1920 if (pt->sample_pwr_events) { in intel_pt_sample()
1950 if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) { in intel_pt_sample()
1956 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) { in intel_pt_sample()
1962 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) { in intel_pt_sample()
1971 if (pt->synth_opts.callchain || pt->synth_opts.thread_stack) in intel_pt_sample()
1978 if (pt->sample_branches) { in intel_pt_sample()
1984 if (pt->synth_opts.last_branch) in intel_pt_sample()
1995 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
2009 state->to_ip == pt->ptss_ip && in intel_pt_sample()
2017 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip) in intel_pt_switch_ip() argument
2019 struct machine *machine = pt->machine; in intel_pt_switch_ip()
2051 if (pt->have_sched_switch == 1) in intel_pt_switch_ip()
2069 static void intel_pt_enable_sync_switch(struct intel_pt *pt) in intel_pt_enable_sync_switch() argument
2073 pt->sync_switch = true; in intel_pt_enable_sync_switch()
2075 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_enable_sync_switch()
2076 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_enable_sync_switch()
2090 struct intel_pt *pt = ptq->pt; in intel_pt_next_time() local
2095 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end; in intel_pt_next_time()
2097 } else if (ptq->sel_idx + 1 < pt->range_cnt) { in intel_pt_next_time()
2101 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start; in intel_pt_next_time()
2129 intel_pt_next_tid(ptq->pt, ptq); in intel_pt_time_filter()
2159 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder() local
2163 if (!pt->kernel_start) { in intel_pt_run_decoder()
2164 pt->kernel_start = machine__kernel_start(pt->machine); in intel_pt_run_decoder()
2165 if (pt->per_cpu_mmaps && in intel_pt_run_decoder()
2166 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) && in intel_pt_run_decoder()
2167 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) && in intel_pt_run_decoder()
2168 !pt->sampling_mode) { in intel_pt_run_decoder()
2169 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip); in intel_pt_run_decoder()
2170 if (pt->switch_ip) { in intel_pt_run_decoder()
2172 pt->switch_ip, pt->ptss_ip); in intel_pt_run_decoder()
2173 intel_pt_enable_sync_switch(pt); in intel_pt_run_decoder()
2190 state->from_ip >= pt->kernel_start) { in intel_pt_run_decoder()
2192 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
2194 if (pt->synth_opts.errors) { in intel_pt_run_decoder()
2207 if (pt->est_tsc && in intel_pt_run_decoder()
2208 (state->from_ip >= pt->kernel_start || !state->from_ip) && in intel_pt_run_decoder()
2209 state->to_ip && state->to_ip < pt->kernel_start) { in intel_pt_run_decoder()
2231 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
2239 static inline int intel_pt_update_queues(struct intel_pt *pt) in intel_pt_update_queues() argument
2241 if (pt->queues.new_data) { in intel_pt_update_queues()
2242 pt->queues.new_data = false; in intel_pt_update_queues()
2243 return intel_pt_setup_queues(pt); in intel_pt_update_queues()
2248 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp) in intel_pt_process_queues() argument
2258 if (!pt->heap.heap_cnt) in intel_pt_process_queues()
2261 if (pt->heap.heap_array[0].ordinal >= timestamp) in intel_pt_process_queues()
2264 queue_nr = pt->heap.heap_array[0].queue_nr; in intel_pt_process_queues()
2265 queue = &pt->queues.queue_array[queue_nr]; in intel_pt_process_queues()
2269 queue_nr, pt->heap.heap_array[0].ordinal, in intel_pt_process_queues()
2272 auxtrace_heap__pop(&pt->heap); in intel_pt_process_queues()
2274 if (pt->heap.heap_cnt) { in intel_pt_process_queues()
2275 ts = pt->heap.heap_array[0].ordinal + 1; in intel_pt_process_queues()
2282 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_queues()
2287 auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
2292 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
2303 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid, in intel_pt_process_timeless_queues() argument
2306 struct auxtrace_queues *queues = &pt->queues; in intel_pt_process_timeless_queues()
2311 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_process_timeless_queues()
2316 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_timeless_queues()
2323 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample) in intel_pt_lost() argument
2325 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu, in intel_pt_lost()
2329 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu) in intel_pt_cpu_to_ptq() argument
2333 if (cpu < 0 || !pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
2336 if ((unsigned)cpu >= pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
2337 i = pt->queues.nr_queues - 1; in intel_pt_cpu_to_ptq()
2341 if (pt->queues.queue_array[i].cpu == cpu) in intel_pt_cpu_to_ptq()
2342 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
2345 if (pt->queues.queue_array[--i].cpu == cpu) in intel_pt_cpu_to_ptq()
2346 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
2349 for (; j < pt->queues.nr_queues; j++) { in intel_pt_cpu_to_ptq()
2350 if (pt->queues.queue_array[j].cpu == cpu) in intel_pt_cpu_to_ptq()
2351 return pt->queues.queue_array[j].priv; in intel_pt_cpu_to_ptq()
2357 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid, in intel_pt_sync_switch() argument
2363 if (!pt->sync_switch) in intel_pt_sync_switch()
2366 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
2381 &pt->tc); in intel_pt_sync_switch()
2382 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
2402 static int intel_pt_process_switch(struct intel_pt *pt, in intel_pt_process_switch() argument
2409 evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id); in intel_pt_process_switch()
2410 if (evsel != pt->switch_evsel) in intel_pt_process_switch()
2418 &pt->tc)); in intel_pt_process_switch()
2420 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_process_switch()
2424 return machine__set_current_tid(pt->machine, cpu, -1, tid); in intel_pt_process_switch()
2427 static int intel_pt_context_switch_in(struct intel_pt *pt, in intel_pt_context_switch_in() argument
2434 if (pt->sync_switch) { in intel_pt_context_switch_in()
2437 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_context_switch_in()
2459 if (machine__get_current_tid(pt->machine, cpu) == tid) in intel_pt_context_switch_in()
2462 return machine__set_current_tid(pt->machine, cpu, pid, tid); in intel_pt_context_switch_in()
2465 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event, in intel_pt_context_switch() argument
2474 if (pt->have_sched_switch == 3) { in intel_pt_context_switch()
2476 return intel_pt_context_switch_in(pt, sample); in intel_pt_context_switch()
2497 &pt->tc)); in intel_pt_context_switch()
2499 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_context_switch()
2503 return machine__set_current_tid(pt->machine, cpu, pid, tid); in intel_pt_context_switch()
2506 static int intel_pt_process_itrace_start(struct intel_pt *pt, in intel_pt_process_itrace_start() argument
2510 if (!pt->per_cpu_mmaps) in intel_pt_process_itrace_start()
2516 perf_time_to_tsc(sample->time, &pt->tc)); in intel_pt_process_itrace_start()
2518 return machine__set_current_tid(pt->machine, sample->cpu, in intel_pt_process_itrace_start()
2528 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_event() local
2542 timestamp = perf_time_to_tsc(sample->time, &pt->tc); in intel_pt_process_event()
2546 if (timestamp || pt->timeless_decoding) { in intel_pt_process_event()
2547 err = intel_pt_update_queues(pt); in intel_pt_process_event()
2552 if (pt->timeless_decoding) { in intel_pt_process_event()
2554 err = intel_pt_process_timeless_queues(pt, in intel_pt_process_event()
2559 err = intel_pt_process_queues(pt, timestamp); in intel_pt_process_event()
2566 pt->synth_opts.errors) { in intel_pt_process_event()
2567 err = intel_pt_lost(pt, sample); in intel_pt_process_event()
2572 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE) in intel_pt_process_event()
2573 err = intel_pt_process_switch(pt, sample); in intel_pt_process_event()
2575 err = intel_pt_process_itrace_start(pt, event, sample); in intel_pt_process_event()
2578 err = intel_pt_context_switch(pt, event, sample); in intel_pt_process_event()
2589 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_flush() local
2599 ret = intel_pt_update_queues(pt); in intel_pt_flush()
2603 if (pt->timeless_decoding) in intel_pt_flush()
2604 return intel_pt_process_timeless_queues(pt, -1, in intel_pt_flush()
2607 return intel_pt_process_queues(pt, MAX_TIMESTAMP); in intel_pt_flush()
2612 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free_events() local
2614 struct auxtrace_queues *queues = &pt->queues; in intel_pt_free_events()
2627 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free() local
2630 auxtrace_heap__free(&pt->heap); in intel_pt_free()
2633 thread__put(pt->unknown_thread); in intel_pt_free()
2634 addr_filters__exit(&pt->filts); in intel_pt_free()
2635 zfree(&pt->filter); in intel_pt_free()
2636 zfree(&pt->time_ranges); in intel_pt_free()
2637 free(pt); in intel_pt_free()
2644 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_auxtrace_event() local
2647 if (!pt->data_queued) { in intel_pt_process_auxtrace_event()
2661 err = auxtrace_queues__add_event(&pt->queues, session, event, in intel_pt_process_auxtrace_event()
2669 intel_pt_dump_event(pt, buffer->data, in intel_pt_process_auxtrace_event()
2732 static struct evsel *intel_pt_evsel(struct intel_pt *pt, in intel_pt_evsel() argument
2738 if (evsel->core.attr.type == pt->pmu_type && evsel->core.ids) in intel_pt_evsel()
2745 static int intel_pt_synth_events(struct intel_pt *pt, in intel_pt_synth_events() argument
2749 struct evsel *evsel = intel_pt_evsel(pt, evlist); in intel_pt_synth_events()
2765 if (pt->timeless_decoding) in intel_pt_synth_events()
2769 if (!pt->per_cpu_mmaps) in intel_pt_synth_events()
2783 if (pt->synth_opts.branches) { in intel_pt_synth_events()
2790 pt->sample_branches = true; in intel_pt_synth_events()
2791 pt->branches_sample_type = attr.sample_type; in intel_pt_synth_events()
2792 pt->branches_id = id; in intel_pt_synth_events()
2797 if (pt->synth_opts.callchain) in intel_pt_synth_events()
2799 if (pt->synth_opts.last_branch) in intel_pt_synth_events()
2802 if (pt->synth_opts.instructions) { in intel_pt_synth_events()
2804 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS) in intel_pt_synth_events()
2806 intel_pt_ns_to_ticks(pt, pt->synth_opts.period); in intel_pt_synth_events()
2808 attr.sample_period = pt->synth_opts.period; in intel_pt_synth_events()
2812 pt->sample_instructions = true; in intel_pt_synth_events()
2813 pt->instructions_sample_type = attr.sample_type; in intel_pt_synth_events()
2814 pt->instructions_id = id; in intel_pt_synth_events()
2821 if (pt->synth_opts.transactions) { in intel_pt_synth_events()
2826 pt->sample_transactions = true; in intel_pt_synth_events()
2827 pt->transactions_sample_type = attr.sample_type; in intel_pt_synth_events()
2828 pt->transactions_id = id; in intel_pt_synth_events()
2836 if (pt->synth_opts.ptwrites) { in intel_pt_synth_events()
2841 pt->sample_ptwrites = true; in intel_pt_synth_events()
2842 pt->ptwrites_sample_type = attr.sample_type; in intel_pt_synth_events()
2843 pt->ptwrites_id = id; in intel_pt_synth_events()
2848 if (pt->synth_opts.pwr_events) { in intel_pt_synth_events()
2849 pt->sample_pwr_events = true; in intel_pt_synth_events()
2850 pt->pwr_events_sample_type = attr.sample_type; in intel_pt_synth_events()
2856 pt->cbr_id = id; in intel_pt_synth_events()
2861 if (pt->synth_opts.pwr_events && (evsel->core.attr.config & 0x10)) { in intel_pt_synth_events()
2866 pt->mwait_id = id; in intel_pt_synth_events()
2874 pt->pwre_id = id; in intel_pt_synth_events()
2882 pt->exstop_id = id; in intel_pt_synth_events()
2890 pt->pwrx_id = id; in intel_pt_synth_events()
2898 static void intel_pt_setup_pebs_events(struct intel_pt *pt) in intel_pt_setup_pebs_events() argument
2902 if (!pt->synth_opts.other_events) in intel_pt_setup_pebs_events()
2905 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_setup_pebs_events()
2907 pt->sample_pebs = true; in intel_pt_setup_pebs_events()
2908 pt->pebs_evsel = evsel; in intel_pt_setup_pebs_events()
2942 struct intel_pt *pt = data; in intel_pt_perf_config() local
2945 pt->mispred_all = perf_config_bool(var, value); in intel_pt_perf_config()
2951 static u64 intel_pt_tsc_start(u64 ns, struct intel_pt *pt) in intel_pt_tsc_start() argument
2955 tsc = perf_time_to_tsc(ns, &pt->tc); in intel_pt_tsc_start()
2958 tm = tsc_to_perf_time(tsc, &pt->tc); in intel_pt_tsc_start()
2965 tm = tsc_to_perf_time(++tsc, &pt->tc); in intel_pt_tsc_start()
2971 static u64 intel_pt_tsc_end(u64 ns, struct intel_pt *pt) in intel_pt_tsc_end() argument
2975 tsc = perf_time_to_tsc(ns, &pt->tc); in intel_pt_tsc_end()
2978 tm = tsc_to_perf_time(tsc, &pt->tc); in intel_pt_tsc_end()
2985 tm = tsc_to_perf_time(--tsc, &pt->tc); in intel_pt_tsc_end()
2990 static int intel_pt_setup_time_ranges(struct intel_pt *pt, in intel_pt_setup_time_ranges() argument
2997 if (!n || !p || pt->timeless_decoding) in intel_pt_setup_time_ranges()
3000 pt->time_ranges = calloc(n, sizeof(struct range)); in intel_pt_setup_time_ranges()
3001 if (!pt->time_ranges) in intel_pt_setup_time_ranges()
3004 pt->range_cnt = n; in intel_pt_setup_time_ranges()
3009 struct range *r = &pt->time_ranges[i]; in intel_pt_setup_time_ranges()
3017 r->start = ts ? intel_pt_tsc_start(ts, pt) : 0; in intel_pt_setup_time_ranges()
3018 r->end = te ? intel_pt_tsc_end(te, pt) : 0; in intel_pt_setup_time_ranges()
3078 struct intel_pt *pt; in intel_pt_process_auxtrace_info() local
3087 pt = zalloc(sizeof(struct intel_pt)); in intel_pt_process_auxtrace_info()
3088 if (!pt) in intel_pt_process_auxtrace_info()
3091 addr_filters__init(&pt->filts); in intel_pt_process_auxtrace_info()
3093 err = perf_config(intel_pt_perf_config, pt); in intel_pt_process_auxtrace_info()
3097 err = auxtrace_queues__init(&pt->queues); in intel_pt_process_auxtrace_info()
3103 pt->session = session; in intel_pt_process_auxtrace_info()
3104 pt->machine = &session->machines.host; /* No kvm support */ in intel_pt_process_auxtrace_info()
3105 pt->auxtrace_type = auxtrace_info->type; in intel_pt_process_auxtrace_info()
3106 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE]; in intel_pt_process_auxtrace_info()
3107 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT]; in intel_pt_process_auxtrace_info()
3108 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT]; in intel_pt_process_auxtrace_info()
3109 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO]; in intel_pt_process_auxtrace_info()
3110 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO]; in intel_pt_process_auxtrace_info()
3111 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT]; in intel_pt_process_auxtrace_info()
3112 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT]; in intel_pt_process_auxtrace_info()
3113 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH]; in intel_pt_process_auxtrace_info()
3114 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE]; in intel_pt_process_auxtrace_info()
3115 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS]; in intel_pt_process_auxtrace_info()
3120 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT]; in intel_pt_process_auxtrace_info()
3121 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS]; in intel_pt_process_auxtrace_info()
3122 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N]; in intel_pt_process_auxtrace_info()
3123 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D]; in intel_pt_process_auxtrace_info()
3124 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT]; in intel_pt_process_auxtrace_info()
3130 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
3157 pt->filter = memdup(filter, len); in intel_pt_process_auxtrace_info()
3158 if (!pt->filter) { in intel_pt_process_auxtrace_info()
3163 mem_bswap_64(pt->filter, len); in intel_pt_process_auxtrace_info()
3164 if (pt->filter[len - 1]) { in intel_pt_process_auxtrace_info()
3169 err = addr_filters__parse_bare_filter(&pt->filts, in intel_pt_process_auxtrace_info()
3174 intel_pt_print_info_str("Filter string", pt->filter); in intel_pt_process_auxtrace_info()
3177 pt->timeless_decoding = intel_pt_timeless_decoding(pt); in intel_pt_process_auxtrace_info()
3178 if (pt->timeless_decoding && !pt->tc.time_mult) in intel_pt_process_auxtrace_info()
3179 pt->tc.time_mult = 1; in intel_pt_process_auxtrace_info()
3180 pt->have_tsc = intel_pt_have_tsc(pt); in intel_pt_process_auxtrace_info()
3181 pt->sampling_mode = false; in intel_pt_process_auxtrace_info()
3182 pt->est_tsc = !pt->timeless_decoding; in intel_pt_process_auxtrace_info()
3184 pt->unknown_thread = thread__new(999999999, 999999999); in intel_pt_process_auxtrace_info()
3185 if (!pt->unknown_thread) { in intel_pt_process_auxtrace_info()
3196 INIT_LIST_HEAD(&pt->unknown_thread->node); in intel_pt_process_auxtrace_info()
3198 err = thread__set_comm(pt->unknown_thread, "unknown", 0); in intel_pt_process_auxtrace_info()
3201 if (thread__init_map_groups(pt->unknown_thread, pt->machine)) { in intel_pt_process_auxtrace_info()
3206 pt->auxtrace.process_event = intel_pt_process_event; in intel_pt_process_auxtrace_info()
3207 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event; in intel_pt_process_auxtrace_info()
3208 pt->auxtrace.flush_events = intel_pt_flush; in intel_pt_process_auxtrace_info()
3209 pt->auxtrace.free_events = intel_pt_free_events; in intel_pt_process_auxtrace_info()
3210 pt->auxtrace.free = intel_pt_free; in intel_pt_process_auxtrace_info()
3211 session->auxtrace = &pt->auxtrace; in intel_pt_process_auxtrace_info()
3216 if (pt->have_sched_switch == 1) { in intel_pt_process_auxtrace_info()
3217 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist); in intel_pt_process_auxtrace_info()
3218 if (!pt->switch_evsel) { in intel_pt_process_auxtrace_info()
3223 } else if (pt->have_sched_switch == 2 && in intel_pt_process_auxtrace_info()
3231 pt->synth_opts = *session->itrace_synth_opts; in intel_pt_process_auxtrace_info()
3233 itrace_synth_opts__set_default(&pt->synth_opts, in intel_pt_process_auxtrace_info()
3237 pt->synth_opts.branches = false; in intel_pt_process_auxtrace_info()
3238 pt->synth_opts.callchain = true; in intel_pt_process_auxtrace_info()
3240 pt->synth_opts.thread_stack = in intel_pt_process_auxtrace_info()
3244 if (pt->synth_opts.log) in intel_pt_process_auxtrace_info()
3248 if (pt->tc.time_mult) { in intel_pt_process_auxtrace_info()
3249 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000); in intel_pt_process_auxtrace_info()
3251 if (!pt->max_non_turbo_ratio) in intel_pt_process_auxtrace_info()
3252 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
3256 pt->max_non_turbo_ratio); in intel_pt_process_auxtrace_info()
3257 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000; in intel_pt_process_auxtrace_info()
3260 err = intel_pt_setup_time_ranges(pt, session->itrace_synth_opts); in intel_pt_process_auxtrace_info()
3264 if (pt->synth_opts.calls) in intel_pt_process_auxtrace_info()
3265 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | in intel_pt_process_auxtrace_info()
3267 if (pt->synth_opts.returns) in intel_pt_process_auxtrace_info()
3268 pt->branches_filter |= PERF_IP_FLAG_RETURN | in intel_pt_process_auxtrace_info()
3271 if (pt->synth_opts.callchain && !symbol_conf.use_callchain) { in intel_pt_process_auxtrace_info()
3275 pt->synth_opts.callchain = false; in intel_pt_process_auxtrace_info()
3279 err = intel_pt_synth_events(pt, session); in intel_pt_process_auxtrace_info()
3283 intel_pt_setup_pebs_events(pt); in intel_pt_process_auxtrace_info()
3285 err = auxtrace_queues__process_index(&pt->queues, session); in intel_pt_process_auxtrace_info()
3289 if (pt->queues.populated) in intel_pt_process_auxtrace_info()
3290 pt->data_queued = true; in intel_pt_process_auxtrace_info()
3292 if (pt->timeless_decoding) in intel_pt_process_auxtrace_info()
3298 thread__zput(pt->unknown_thread); in intel_pt_process_auxtrace_info()
3301 auxtrace_queues__free(&pt->queues); in intel_pt_process_auxtrace_info()
3304 addr_filters__exit(&pt->filts); in intel_pt_process_auxtrace_info()
3305 zfree(&pt->filter); in intel_pt_process_auxtrace_info()
3306 zfree(&pt->time_ranges); in intel_pt_process_auxtrace_info()
3307 free(pt); in intel_pt_process_auxtrace_info()