Lines Matching refs:pt
132 struct intel_pt *pt; member
163 static void intel_pt_dump(struct intel_pt *pt __maybe_unused, in intel_pt_dump()
202 static void intel_pt_dump_event(struct intel_pt *pt, unsigned char *buf, in intel_pt_dump_event() argument
206 intel_pt_dump(pt, buf, len); in intel_pt_dump_event()
209 static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a, in intel_pt_do_fix_overlap() argument
216 pt->have_tsc, &consecutive); in intel_pt_do_fix_overlap()
240 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
253 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_trace()
260 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_trace()
262 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_trace()
416 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
432 if (*ip >= ptq->pt->kernel_start) in intel_pt_walk_next_insn()
441 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
543 static bool intel_pt_match_pgd_ip(struct intel_pt *pt, uint64_t ip, in intel_pt_match_pgd_ip() argument
551 list_for_each_entry(filt, &pt->filts.head, list) { in intel_pt_match_pgd_ip()
589 if (ip >= ptq->pt->kernel_start) in __intel_pt_pgd_ip()
590 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
603 return intel_pt_match_pgd_ip(ptq->pt, ip, offset, in __intel_pt_pgd_ip()
612 static bool intel_pt_get_config(struct intel_pt *pt, in intel_pt_get_config() argument
615 if (attr->type == pt->pmu_type) { in intel_pt_get_config()
624 static bool intel_pt_exclude_kernel(struct intel_pt *pt) in intel_pt_exclude_kernel() argument
628 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_exclude_kernel()
629 if (intel_pt_get_config(pt, &evsel->attr, NULL) && in intel_pt_exclude_kernel()
636 static bool intel_pt_return_compression(struct intel_pt *pt) in intel_pt_return_compression() argument
641 if (!pt->noretcomp_bit) in intel_pt_return_compression()
644 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_return_compression()
645 if (intel_pt_get_config(pt, &evsel->attr, &config) && in intel_pt_return_compression()
646 (config & pt->noretcomp_bit)) in intel_pt_return_compression()
652 static bool intel_pt_branch_enable(struct intel_pt *pt) in intel_pt_branch_enable() argument
657 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_branch_enable()
658 if (intel_pt_get_config(pt, &evsel->attr, &config) && in intel_pt_branch_enable()
665 static unsigned int intel_pt_mtc_period(struct intel_pt *pt) in intel_pt_mtc_period() argument
671 if (!pt->mtc_freq_bits) in intel_pt_mtc_period()
674 for (shift = 0, config = pt->mtc_freq_bits; !(config & 1); shift++) in intel_pt_mtc_period()
677 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_mtc_period()
678 if (intel_pt_get_config(pt, &evsel->attr, &config)) in intel_pt_mtc_period()
679 return (config & pt->mtc_freq_bits) >> shift; in intel_pt_mtc_period()
684 static bool intel_pt_timeless_decoding(struct intel_pt *pt) in intel_pt_timeless_decoding() argument
690 if (!pt->tsc_bit || !pt->cap_user_time_zero) in intel_pt_timeless_decoding()
693 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_timeless_decoding()
696 if (intel_pt_get_config(pt, &evsel->attr, &config)) { in intel_pt_timeless_decoding()
697 if (config & pt->tsc_bit) in intel_pt_timeless_decoding()
706 static bool intel_pt_tracing_kernel(struct intel_pt *pt) in intel_pt_tracing_kernel() argument
710 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_tracing_kernel()
711 if (intel_pt_get_config(pt, &evsel->attr, NULL) && in intel_pt_tracing_kernel()
718 static bool intel_pt_have_tsc(struct intel_pt *pt) in intel_pt_have_tsc() argument
724 if (!pt->tsc_bit) in intel_pt_have_tsc()
727 evlist__for_each_entry(pt->session->evlist, evsel) { in intel_pt_have_tsc()
728 if (intel_pt_get_config(pt, &evsel->attr, &config)) { in intel_pt_have_tsc()
729 if (config & pt->tsc_bit) in intel_pt_have_tsc()
738 static u64 intel_pt_ns_to_ticks(const struct intel_pt *pt, u64 ns) in intel_pt_ns_to_ticks() argument
742 quot = ns / pt->tc.time_mult; in intel_pt_ns_to_ticks()
743 rem = ns % pt->tc.time_mult; in intel_pt_ns_to_ticks()
744 return (quot << pt->tc.time_shift) + (rem << pt->tc.time_shift) / in intel_pt_ns_to_ticks()
745 pt->tc.time_mult; in intel_pt_ns_to_ticks()
748 static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, in intel_pt_alloc_queue() argument
752 struct perf_env *env = pt->machine->env; in intel_pt_alloc_queue()
759 if (pt->synth_opts.callchain) { in intel_pt_alloc_queue()
762 sz += pt->synth_opts.callchain_sz * sizeof(u64); in intel_pt_alloc_queue()
768 if (pt->synth_opts.last_branch) { in intel_pt_alloc_queue()
771 sz += pt->synth_opts.last_branch_sz * in intel_pt_alloc_queue()
785 ptq->pt = pt; in intel_pt_alloc_queue()
787 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
796 params.return_compression = intel_pt_return_compression(pt); in intel_pt_alloc_queue()
797 params.branch_enable = intel_pt_branch_enable(pt); in intel_pt_alloc_queue()
798 params.max_non_turbo_ratio = pt->max_non_turbo_ratio; in intel_pt_alloc_queue()
799 params.mtc_period = intel_pt_mtc_period(pt); in intel_pt_alloc_queue()
800 params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n; in intel_pt_alloc_queue()
801 params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d; in intel_pt_alloc_queue()
803 if (pt->filts.cnt > 0) in intel_pt_alloc_queue()
806 if (pt->synth_opts.instructions) { in intel_pt_alloc_queue()
807 if (pt->synth_opts.period) { in intel_pt_alloc_queue()
808 switch (pt->synth_opts.period_type) { in intel_pt_alloc_queue()
812 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
816 params.period = pt->synth_opts.period; in intel_pt_alloc_queue()
820 params.period = intel_pt_ns_to_ticks(pt, in intel_pt_alloc_queue()
821 pt->synth_opts.period); in intel_pt_alloc_queue()
867 static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt, in intel_pt_set_pid_tid_cpu() argument
872 if (queue->tid == -1 || pt->have_sched_switch) { in intel_pt_set_pid_tid_cpu()
873 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
878 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
913 static int intel_pt_setup_queue(struct intel_pt *pt, in intel_pt_setup_queue() argument
923 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
932 if (pt->sampling_mode && !pt->snapshot_mode && in intel_pt_setup_queue()
933 pt->timeless_decoding) in intel_pt_setup_queue()
936 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
945 if (pt->timeless_decoding) in intel_pt_setup_queue()
971 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
980 static int intel_pt_setup_queues(struct intel_pt *pt) in intel_pt_setup_queues() argument
985 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_setup_queues()
986 ret = intel_pt_setup_queue(pt, &pt->queues.queue_array[i], i); in intel_pt_setup_queues()
1004 nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos; in intel_pt_copy_last_branch_rb()
1009 if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) { in intel_pt_copy_last_branch_rb()
1029 ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz; in intel_pt_update_last_branch_rb()
1039 be->flags.mispred = ptq->pt->mispred_all; in intel_pt_update_last_branch_rb()
1041 if (bs->nr < ptq->pt->synth_opts.last_branch_sz) in intel_pt_update_last_branch_rb()
1045 static inline bool intel_pt_skip_event(struct intel_pt *pt) in intel_pt_skip_event() argument
1047 return pt->synth_opts.initial_skip && in intel_pt_skip_event()
1048 pt->num_events++ < pt->synth_opts.initial_skip; in intel_pt_skip_event()
1051 static void intel_pt_prep_b_sample(struct intel_pt *pt, in intel_pt_prep_b_sample() argument
1060 if (!pt->timeless_decoding) in intel_pt_prep_b_sample()
1061 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1082 static inline int intel_pt_opt_inject(struct intel_pt *pt, in intel_pt_opt_inject() argument
1086 if (!pt->synth_opts.inject) in intel_pt_opt_inject()
1092 static int intel_pt_deliver_synth_b_event(struct intel_pt *pt, in intel_pt_deliver_synth_b_event() argument
1098 ret = intel_pt_opt_inject(pt, event, sample, type); in intel_pt_deliver_synth_b_event()
1102 ret = perf_session__deliver_synth_event(pt->session, event, sample); in intel_pt_deliver_synth_b_event()
1111 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample() local
1119 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1122 if (intel_pt_skip_event(pt)) in intel_pt_synth_branch_sample()
1125 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1127 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1128 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1134 if (pt->synth_opts.last_branch && sort__mode == SORT_MODE__BRANCH) { in intel_pt_synth_branch_sample()
1145 return intel_pt_deliver_synth_b_event(pt, event, &sample, in intel_pt_synth_branch_sample()
1146 pt->branches_sample_type); in intel_pt_synth_branch_sample()
1149 static void intel_pt_prep_sample(struct intel_pt *pt, in intel_pt_prep_sample() argument
1154 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1156 if (pt->synth_opts.callchain) { in intel_pt_prep_sample()
1158 pt->synth_opts.callchain_sz, sample->ip); in intel_pt_prep_sample()
1162 if (pt->synth_opts.last_branch) { in intel_pt_prep_sample()
1168 static inline int intel_pt_deliver_synth_event(struct intel_pt *pt, in intel_pt_deliver_synth_event() argument
1176 ret = intel_pt_deliver_synth_b_event(pt, event, sample, type); in intel_pt_deliver_synth_event()
1178 if (pt->synth_opts.last_branch) in intel_pt_deliver_synth_event()
1186 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample() local
1190 if (intel_pt_skip_event(pt)) in intel_pt_synth_instruction_sample()
1193 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1195 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1196 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1201 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_instruction_sample()
1202 pt->instructions_sample_type); in intel_pt_synth_instruction_sample()
1207 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample() local
1211 if (intel_pt_skip_event(pt)) in intel_pt_synth_transaction_sample()
1214 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1216 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1217 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1219 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_transaction_sample()
1220 pt->transactions_sample_type); in intel_pt_synth_transaction_sample()
1223 static void intel_pt_prep_p_sample(struct intel_pt *pt, in intel_pt_prep_p_sample() argument
1228 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1240 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample() local
1245 if (intel_pt_skip_event(pt)) in intel_pt_synth_ptwrite_sample()
1248 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1250 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1251 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1260 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_ptwrite_sample()
1261 pt->ptwrites_sample_type); in intel_pt_synth_ptwrite_sample()
1266 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample() local
1272 if (intel_pt_skip_event(pt)) in intel_pt_synth_cbr_sample()
1275 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1277 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1278 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1280 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1282 raw.freq = cpu_to_le32(raw.cbr * pt->cbr2khz); in intel_pt_synth_cbr_sample()
1288 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_cbr_sample()
1289 pt->pwr_events_sample_type); in intel_pt_synth_cbr_sample()
1294 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample() local
1299 if (intel_pt_skip_event(pt)) in intel_pt_synth_mwait_sample()
1302 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
1304 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1305 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1313 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_mwait_sample()
1314 pt->pwr_events_sample_type); in intel_pt_synth_mwait_sample()
1319 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample() local
1324 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwre_sample()
1327 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
1329 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1330 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1338 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_pwre_sample()
1339 pt->pwr_events_sample_type); in intel_pt_synth_pwre_sample()
1344 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample() local
1349 if (intel_pt_skip_event(pt)) in intel_pt_synth_exstop_sample()
1352 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
1354 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1355 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1363 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_exstop_sample()
1364 pt->pwr_events_sample_type); in intel_pt_synth_exstop_sample()
1369 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample() local
1374 if (intel_pt_skip_event(pt)) in intel_pt_synth_pwrx_sample()
1377 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
1379 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1380 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1388 return intel_pt_deliver_synth_event(pt, ptq, event, &sample, in intel_pt_synth_pwrx_sample()
1389 pt->pwr_events_sample_type); in intel_pt_synth_pwrx_sample()
1392 static int intel_pt_synth_error(struct intel_pt *pt, int code, int cpu, in intel_pt_synth_error() argument
1404 err = perf_session__deliver_synth_event(pt->session, &event, NULL); in intel_pt_synth_error()
1412 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
1423 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
1425 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
1426 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_next_tid()
1435 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip() local
1437 return ip == pt->switch_ip && in intel_pt_is_switch_ip()
1450 struct intel_pt *pt = ptq->pt; in intel_pt_sample() local
1458 if (pt->sample_pwr_events && (state->type & INTEL_PT_PWR_EVT)) { in intel_pt_sample()
1486 if (pt->sample_instructions && (state->type & INTEL_PT_INSTRUCTION)) { in intel_pt_sample()
1492 if (pt->sample_transactions && (state->type & INTEL_PT_TRANSACTION)) { in intel_pt_sample()
1498 if (pt->sample_ptwrites && (state->type & INTEL_PT_PTW)) { in intel_pt_sample()
1507 if (pt->synth_opts.callchain || pt->synth_opts.thread_stack) in intel_pt_sample()
1514 if (pt->sample_branches) { in intel_pt_sample()
1520 if (pt->synth_opts.last_branch) in intel_pt_sample()
1531 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
1545 state->to_ip == pt->ptss_ip && in intel_pt_sample()
1553 static u64 intel_pt_switch_ip(struct intel_pt *pt, u64 *ptss_ip) in intel_pt_switch_ip() argument
1555 struct machine *machine = pt->machine; in intel_pt_switch_ip()
1587 if (pt->have_sched_switch == 1) in intel_pt_switch_ip()
1605 static void intel_pt_enable_sync_switch(struct intel_pt *pt) in intel_pt_enable_sync_switch() argument
1609 pt->sync_switch = true; in intel_pt_enable_sync_switch()
1611 for (i = 0; i < pt->queues.nr_queues; i++) { in intel_pt_enable_sync_switch()
1612 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_enable_sync_switch()
1623 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder() local
1626 if (!pt->kernel_start) { in intel_pt_run_decoder()
1627 pt->kernel_start = machine__kernel_start(pt->machine); in intel_pt_run_decoder()
1628 if (pt->per_cpu_mmaps && in intel_pt_run_decoder()
1629 (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) && in intel_pt_run_decoder()
1630 !pt->timeless_decoding && intel_pt_tracing_kernel(pt) && in intel_pt_run_decoder()
1631 !pt->sampling_mode) { in intel_pt_run_decoder()
1632 pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip); in intel_pt_run_decoder()
1633 if (pt->switch_ip) { in intel_pt_run_decoder()
1635 pt->switch_ip, pt->ptss_ip); in intel_pt_run_decoder()
1636 intel_pt_enable_sync_switch(pt); in intel_pt_run_decoder()
1653 state->from_ip >= pt->kernel_start) { in intel_pt_run_decoder()
1655 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
1657 if (pt->synth_opts.errors) { in intel_pt_run_decoder()
1658 err = intel_pt_synth_error(pt, state->err, in intel_pt_run_decoder()
1673 if (pt->est_tsc && in intel_pt_run_decoder()
1674 (state->from_ip >= pt->kernel_start || !state->from_ip) && in intel_pt_run_decoder()
1675 state->to_ip && state->to_ip < pt->kernel_start) { in intel_pt_run_decoder()
1691 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
1699 static inline int intel_pt_update_queues(struct intel_pt *pt) in intel_pt_update_queues() argument
1701 if (pt->queues.new_data) { in intel_pt_update_queues()
1702 pt->queues.new_data = false; in intel_pt_update_queues()
1703 return intel_pt_setup_queues(pt); in intel_pt_update_queues()
1708 static int intel_pt_process_queues(struct intel_pt *pt, u64 timestamp) in intel_pt_process_queues() argument
1718 if (!pt->heap.heap_cnt) in intel_pt_process_queues()
1721 if (pt->heap.heap_array[0].ordinal >= timestamp) in intel_pt_process_queues()
1724 queue_nr = pt->heap.heap_array[0].queue_nr; in intel_pt_process_queues()
1725 queue = &pt->queues.queue_array[queue_nr]; in intel_pt_process_queues()
1729 queue_nr, pt->heap.heap_array[0].ordinal, in intel_pt_process_queues()
1732 auxtrace_heap__pop(&pt->heap); in intel_pt_process_queues()
1734 if (pt->heap.heap_cnt) { in intel_pt_process_queues()
1735 ts = pt->heap.heap_array[0].ordinal + 1; in intel_pt_process_queues()
1742 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_queues()
1747 auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
1752 ret = auxtrace_heap__add(&pt->heap, queue_nr, ts); in intel_pt_process_queues()
1763 static int intel_pt_process_timeless_queues(struct intel_pt *pt, pid_t tid, in intel_pt_process_timeless_queues() argument
1766 struct auxtrace_queues *queues = &pt->queues; in intel_pt_process_timeless_queues()
1771 struct auxtrace_queue *queue = &pt->queues.queue_array[i]; in intel_pt_process_timeless_queues()
1776 intel_pt_set_pid_tid_cpu(pt, queue); in intel_pt_process_timeless_queues()
1783 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample) in intel_pt_lost() argument
1785 return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu, in intel_pt_lost()
1789 static struct intel_pt_queue *intel_pt_cpu_to_ptq(struct intel_pt *pt, int cpu) in intel_pt_cpu_to_ptq() argument
1793 if (cpu < 0 || !pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
1796 if ((unsigned)cpu >= pt->queues.nr_queues) in intel_pt_cpu_to_ptq()
1797 i = pt->queues.nr_queues - 1; in intel_pt_cpu_to_ptq()
1801 if (pt->queues.queue_array[i].cpu == cpu) in intel_pt_cpu_to_ptq()
1802 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
1805 if (pt->queues.queue_array[--i].cpu == cpu) in intel_pt_cpu_to_ptq()
1806 return pt->queues.queue_array[i].priv; in intel_pt_cpu_to_ptq()
1809 for (; j < pt->queues.nr_queues; j++) { in intel_pt_cpu_to_ptq()
1810 if (pt->queues.queue_array[j].cpu == cpu) in intel_pt_cpu_to_ptq()
1811 return pt->queues.queue_array[j].priv; in intel_pt_cpu_to_ptq()
1817 static int intel_pt_sync_switch(struct intel_pt *pt, int cpu, pid_t tid, in intel_pt_sync_switch() argument
1823 if (!pt->sync_switch) in intel_pt_sync_switch()
1826 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
1842 &pt->tc); in intel_pt_sync_switch()
1843 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
1862 static int intel_pt_process_switch(struct intel_pt *pt, in intel_pt_process_switch() argument
1869 evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id); in intel_pt_process_switch()
1870 if (evsel != pt->switch_evsel) in intel_pt_process_switch()
1878 &pt->tc)); in intel_pt_process_switch()
1880 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_process_switch()
1884 return machine__set_current_tid(pt->machine, cpu, -1, tid); in intel_pt_process_switch()
1887 static int intel_pt_context_switch(struct intel_pt *pt, union perf_event *event, in intel_pt_context_switch() argument
1896 if (pt->have_sched_switch == 3) { in intel_pt_context_switch()
1919 &pt->tc)); in intel_pt_context_switch()
1921 ret = intel_pt_sync_switch(pt, cpu, tid, sample->time); in intel_pt_context_switch()
1925 return machine__set_current_tid(pt->machine, cpu, pid, tid); in intel_pt_context_switch()
1928 static int intel_pt_process_itrace_start(struct intel_pt *pt, in intel_pt_process_itrace_start() argument
1932 if (!pt->per_cpu_mmaps) in intel_pt_process_itrace_start()
1938 perf_time_to_tsc(sample->time, &pt->tc)); in intel_pt_process_itrace_start()
1940 return machine__set_current_tid(pt->machine, sample->cpu, in intel_pt_process_itrace_start()
1950 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_event() local
1964 timestamp = perf_time_to_tsc(sample->time, &pt->tc); in intel_pt_process_event()
1968 if (timestamp || pt->timeless_decoding) { in intel_pt_process_event()
1969 err = intel_pt_update_queues(pt); in intel_pt_process_event()
1974 if (pt->timeless_decoding) { in intel_pt_process_event()
1976 err = intel_pt_process_timeless_queues(pt, in intel_pt_process_event()
1981 err = intel_pt_process_queues(pt, timestamp); in intel_pt_process_event()
1988 pt->synth_opts.errors) { in intel_pt_process_event()
1989 err = intel_pt_lost(pt, sample); in intel_pt_process_event()
1994 if (pt->switch_evsel && event->header.type == PERF_RECORD_SAMPLE) in intel_pt_process_event()
1995 err = intel_pt_process_switch(pt, sample); in intel_pt_process_event()
1997 err = intel_pt_process_itrace_start(pt, event, sample); in intel_pt_process_event()
2000 err = intel_pt_context_switch(pt, event, sample); in intel_pt_process_event()
2011 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_flush() local
2021 ret = intel_pt_update_queues(pt); in intel_pt_flush()
2025 if (pt->timeless_decoding) in intel_pt_flush()
2026 return intel_pt_process_timeless_queues(pt, -1, in intel_pt_flush()
2029 return intel_pt_process_queues(pt, MAX_TIMESTAMP); in intel_pt_flush()
2034 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free_events() local
2036 struct auxtrace_queues *queues = &pt->queues; in intel_pt_free_events()
2049 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_free() local
2052 auxtrace_heap__free(&pt->heap); in intel_pt_free()
2055 thread__put(pt->unknown_thread); in intel_pt_free()
2056 addr_filters__exit(&pt->filts); in intel_pt_free()
2057 zfree(&pt->filter); in intel_pt_free()
2058 free(pt); in intel_pt_free()
2065 struct intel_pt *pt = container_of(session->auxtrace, struct intel_pt, in intel_pt_process_auxtrace_event() local
2068 if (!pt->data_queued) { in intel_pt_process_auxtrace_event()
2082 err = auxtrace_queues__add_event(&pt->queues, session, event, in intel_pt_process_auxtrace_event()
2090 intel_pt_dump_event(pt, buffer->data, in intel_pt_process_auxtrace_event()
2153 static struct perf_evsel *intel_pt_evsel(struct intel_pt *pt, in intel_pt_evsel() argument
2159 if (evsel->attr.type == pt->pmu_type && evsel->ids) in intel_pt_evsel()
2166 static int intel_pt_synth_events(struct intel_pt *pt, in intel_pt_synth_events() argument
2170 struct perf_evsel *evsel = intel_pt_evsel(pt, evlist); in intel_pt_synth_events()
2186 if (pt->timeless_decoding) in intel_pt_synth_events()
2190 if (!pt->per_cpu_mmaps) in intel_pt_synth_events()
2204 if (pt->synth_opts.branches) { in intel_pt_synth_events()
2211 pt->sample_branches = true; in intel_pt_synth_events()
2212 pt->branches_sample_type = attr.sample_type; in intel_pt_synth_events()
2213 pt->branches_id = id; in intel_pt_synth_events()
2218 if (pt->synth_opts.callchain) in intel_pt_synth_events()
2220 if (pt->synth_opts.last_branch) in intel_pt_synth_events()
2223 if (pt->synth_opts.instructions) { in intel_pt_synth_events()
2225 if (pt->synth_opts.period_type == PERF_ITRACE_PERIOD_NANOSECS) in intel_pt_synth_events()
2227 intel_pt_ns_to_ticks(pt, pt->synth_opts.period); in intel_pt_synth_events()
2229 attr.sample_period = pt->synth_opts.period; in intel_pt_synth_events()
2233 pt->sample_instructions = true; in intel_pt_synth_events()
2234 pt->instructions_sample_type = attr.sample_type; in intel_pt_synth_events()
2235 pt->instructions_id = id; in intel_pt_synth_events()
2242 if (pt->synth_opts.transactions) { in intel_pt_synth_events()
2247 pt->sample_transactions = true; in intel_pt_synth_events()
2248 pt->transactions_sample_type = attr.sample_type; in intel_pt_synth_events()
2249 pt->transactions_id = id; in intel_pt_synth_events()
2257 if (pt->synth_opts.ptwrites) { in intel_pt_synth_events()
2262 pt->sample_ptwrites = true; in intel_pt_synth_events()
2263 pt->ptwrites_sample_type = attr.sample_type; in intel_pt_synth_events()
2264 pt->ptwrites_id = id; in intel_pt_synth_events()
2269 if (pt->synth_opts.pwr_events) { in intel_pt_synth_events()
2270 pt->sample_pwr_events = true; in intel_pt_synth_events()
2271 pt->pwr_events_sample_type = attr.sample_type; in intel_pt_synth_events()
2277 pt->cbr_id = id; in intel_pt_synth_events()
2282 if (pt->synth_opts.pwr_events && (evsel->attr.config & 0x10)) { in intel_pt_synth_events()
2287 pt->mwait_id = id; in intel_pt_synth_events()
2295 pt->pwre_id = id; in intel_pt_synth_events()
2303 pt->exstop_id = id; in intel_pt_synth_events()
2311 pt->pwrx_id = id; in intel_pt_synth_events()
2347 struct intel_pt *pt = data; in intel_pt_perf_config() local
2350 pt->mispred_all = perf_config_bool(var, value); in intel_pt_perf_config()
2404 struct intel_pt *pt; in intel_pt_process_auxtrace_info() local
2413 pt = zalloc(sizeof(struct intel_pt)); in intel_pt_process_auxtrace_info()
2414 if (!pt) in intel_pt_process_auxtrace_info()
2417 addr_filters__init(&pt->filts); in intel_pt_process_auxtrace_info()
2419 err = perf_config(intel_pt_perf_config, pt); in intel_pt_process_auxtrace_info()
2423 err = auxtrace_queues__init(&pt->queues); in intel_pt_process_auxtrace_info()
2429 pt->session = session; in intel_pt_process_auxtrace_info()
2430 pt->machine = &session->machines.host; /* No kvm support */ in intel_pt_process_auxtrace_info()
2431 pt->auxtrace_type = auxtrace_info->type; in intel_pt_process_auxtrace_info()
2432 pt->pmu_type = auxtrace_info->priv[INTEL_PT_PMU_TYPE]; in intel_pt_process_auxtrace_info()
2433 pt->tc.time_shift = auxtrace_info->priv[INTEL_PT_TIME_SHIFT]; in intel_pt_process_auxtrace_info()
2434 pt->tc.time_mult = auxtrace_info->priv[INTEL_PT_TIME_MULT]; in intel_pt_process_auxtrace_info()
2435 pt->tc.time_zero = auxtrace_info->priv[INTEL_PT_TIME_ZERO]; in intel_pt_process_auxtrace_info()
2436 pt->cap_user_time_zero = auxtrace_info->priv[INTEL_PT_CAP_USER_TIME_ZERO]; in intel_pt_process_auxtrace_info()
2437 pt->tsc_bit = auxtrace_info->priv[INTEL_PT_TSC_BIT]; in intel_pt_process_auxtrace_info()
2438 pt->noretcomp_bit = auxtrace_info->priv[INTEL_PT_NORETCOMP_BIT]; in intel_pt_process_auxtrace_info()
2439 pt->have_sched_switch = auxtrace_info->priv[INTEL_PT_HAVE_SCHED_SWITCH]; in intel_pt_process_auxtrace_info()
2440 pt->snapshot_mode = auxtrace_info->priv[INTEL_PT_SNAPSHOT_MODE]; in intel_pt_process_auxtrace_info()
2441 pt->per_cpu_mmaps = auxtrace_info->priv[INTEL_PT_PER_CPU_MMAPS]; in intel_pt_process_auxtrace_info()
2446 pt->mtc_bit = auxtrace_info->priv[INTEL_PT_MTC_BIT]; in intel_pt_process_auxtrace_info()
2447 pt->mtc_freq_bits = auxtrace_info->priv[INTEL_PT_MTC_FREQ_BITS]; in intel_pt_process_auxtrace_info()
2448 pt->tsc_ctc_ratio_n = auxtrace_info->priv[INTEL_PT_TSC_CTC_N]; in intel_pt_process_auxtrace_info()
2449 pt->tsc_ctc_ratio_d = auxtrace_info->priv[INTEL_PT_TSC_CTC_D]; in intel_pt_process_auxtrace_info()
2450 pt->cyc_bit = auxtrace_info->priv[INTEL_PT_CYC_BIT]; in intel_pt_process_auxtrace_info()
2456 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
2483 pt->filter = memdup(filter, len); in intel_pt_process_auxtrace_info()
2484 if (!pt->filter) { in intel_pt_process_auxtrace_info()
2489 mem_bswap_64(pt->filter, len); in intel_pt_process_auxtrace_info()
2490 if (pt->filter[len - 1]) { in intel_pt_process_auxtrace_info()
2495 err = addr_filters__parse_bare_filter(&pt->filts, in intel_pt_process_auxtrace_info()
2500 intel_pt_print_info_str("Filter string", pt->filter); in intel_pt_process_auxtrace_info()
2503 pt->timeless_decoding = intel_pt_timeless_decoding(pt); in intel_pt_process_auxtrace_info()
2504 pt->have_tsc = intel_pt_have_tsc(pt); in intel_pt_process_auxtrace_info()
2505 pt->sampling_mode = false; in intel_pt_process_auxtrace_info()
2506 pt->est_tsc = !pt->timeless_decoding; in intel_pt_process_auxtrace_info()
2508 pt->unknown_thread = thread__new(999999999, 999999999); in intel_pt_process_auxtrace_info()
2509 if (!pt->unknown_thread) { in intel_pt_process_auxtrace_info()
2520 INIT_LIST_HEAD(&pt->unknown_thread->node); in intel_pt_process_auxtrace_info()
2522 err = thread__set_comm(pt->unknown_thread, "unknown", 0); in intel_pt_process_auxtrace_info()
2525 if (thread__init_map_groups(pt->unknown_thread, pt->machine)) { in intel_pt_process_auxtrace_info()
2530 pt->auxtrace.process_event = intel_pt_process_event; in intel_pt_process_auxtrace_info()
2531 pt->auxtrace.process_auxtrace_event = intel_pt_process_auxtrace_event; in intel_pt_process_auxtrace_info()
2532 pt->auxtrace.flush_events = intel_pt_flush; in intel_pt_process_auxtrace_info()
2533 pt->auxtrace.free_events = intel_pt_free_events; in intel_pt_process_auxtrace_info()
2534 pt->auxtrace.free = intel_pt_free; in intel_pt_process_auxtrace_info()
2535 session->auxtrace = &pt->auxtrace; in intel_pt_process_auxtrace_info()
2540 if (pt->have_sched_switch == 1) { in intel_pt_process_auxtrace_info()
2541 pt->switch_evsel = intel_pt_find_sched_switch(session->evlist); in intel_pt_process_auxtrace_info()
2542 if (!pt->switch_evsel) { in intel_pt_process_auxtrace_info()
2547 } else if (pt->have_sched_switch == 2 && in intel_pt_process_auxtrace_info()
2555 pt->synth_opts = *session->itrace_synth_opts; in intel_pt_process_auxtrace_info()
2557 itrace_synth_opts__set_default(&pt->synth_opts); in intel_pt_process_auxtrace_info()
2559 pt->synth_opts.branches = false; in intel_pt_process_auxtrace_info()
2560 pt->synth_opts.callchain = true; in intel_pt_process_auxtrace_info()
2563 pt->synth_opts.thread_stack = in intel_pt_process_auxtrace_info()
2567 if (pt->synth_opts.log) in intel_pt_process_auxtrace_info()
2571 if (pt->tc.time_mult) { in intel_pt_process_auxtrace_info()
2572 u64 tsc_freq = intel_pt_ns_to_ticks(pt, 1000000000); in intel_pt_process_auxtrace_info()
2574 if (!pt->max_non_turbo_ratio) in intel_pt_process_auxtrace_info()
2575 pt->max_non_turbo_ratio = in intel_pt_process_auxtrace_info()
2579 pt->max_non_turbo_ratio); in intel_pt_process_auxtrace_info()
2580 pt->cbr2khz = tsc_freq / pt->max_non_turbo_ratio / 1000; in intel_pt_process_auxtrace_info()
2583 if (pt->synth_opts.calls) in intel_pt_process_auxtrace_info()
2584 pt->branches_filter |= PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | in intel_pt_process_auxtrace_info()
2586 if (pt->synth_opts.returns) in intel_pt_process_auxtrace_info()
2587 pt->branches_filter |= PERF_IP_FLAG_RETURN | in intel_pt_process_auxtrace_info()
2590 if (pt->synth_opts.callchain && !symbol_conf.use_callchain) { in intel_pt_process_auxtrace_info()
2594 pt->synth_opts.callchain = false; in intel_pt_process_auxtrace_info()
2598 err = intel_pt_synth_events(pt, session); in intel_pt_process_auxtrace_info()
2602 err = auxtrace_queues__process_index(&pt->queues, session); in intel_pt_process_auxtrace_info()
2606 if (pt->queues.populated) in intel_pt_process_auxtrace_info()
2607 pt->data_queued = true; in intel_pt_process_auxtrace_info()
2609 if (pt->timeless_decoding) in intel_pt_process_auxtrace_info()
2615 thread__zput(pt->unknown_thread); in intel_pt_process_auxtrace_info()
2618 auxtrace_queues__free(&pt->queues); in intel_pt_process_auxtrace_info()
2621 addr_filters__exit(&pt->filts); in intel_pt_process_auxtrace_info()
2622 zfree(&pt->filter); in intel_pt_process_auxtrace_info()
2623 free(pt); in intel_pt_process_auxtrace_info()