Lines Matching refs:ptq
315 struct intel_pt_queue *ptq = data; in intel_pt_findnew_vmcs_info() local
316 struct intel_pt *pt = ptq->pt; in intel_pt_findnew_vmcs_info()
363 static int intel_pt_get_buffer(struct intel_pt_queue *ptq, in intel_pt_get_buffer() argument
371 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_buffer()
378 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_buffer()
380 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_buffer()
403 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq, in intel_pt_lookahead_drop_buffer() argument
406 if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer) in intel_pt_lookahead_drop_buffer()
416 struct intel_pt_queue *ptq = data; in intel_pt_lookahead() local
417 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_lookahead()
418 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_lookahead()
422 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_lookahead()
431 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b); in intel_pt_lookahead()
436 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
439 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
449 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
450 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
461 struct intel_pt_queue *ptq = data; in intel_pt_get_trace() local
462 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_get_trace()
463 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_get_trace()
467 if (ptq->stop) { in intel_pt_get_trace()
472 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
482 ptq->buffer = buffer; in intel_pt_get_trace()
484 err = intel_pt_get_buffer(ptq, buffer, old_buffer, b); in intel_pt_get_trace()
488 if (ptq->step_through_buffers) in intel_pt_get_trace()
489 ptq->stop = true; in intel_pt_get_trace()
494 ptq->old_buffer = buffer; in intel_pt_get_trace()
634 static inline u8 intel_pt_nr_cpumode(struct intel_pt_queue *ptq, uint64_t ip, bool nr) in intel_pt_nr_cpumode() argument
642 return ip >= ptq->pt->kernel_start ? in intel_pt_nr_cpumode()
647 static inline u8 intel_pt_cpumode(struct intel_pt_queue *ptq, uint64_t from_ip, uint64_t to_ip) in intel_pt_cpumode() argument
651 return intel_pt_nr_cpumode(ptq, from_ip, ptq->state->from_nr); in intel_pt_cpumode()
652 return intel_pt_nr_cpumode(ptq, to_ip, ptq->state->to_nr); in intel_pt_cpumode()
655 static int intel_pt_get_guest(struct intel_pt_queue *ptq) in intel_pt_get_guest() argument
657 struct machines *machines = &ptq->pt->session->machines; in intel_pt_get_guest()
659 pid_t pid = ptq->pid <= 0 ? DEFAULT_GUEST_KERNEL_ID : ptq->pid; in intel_pt_get_guest()
661 if (ptq->guest_machine && pid == ptq->guest_machine_pid) in intel_pt_get_guest()
664 ptq->guest_machine = NULL; in intel_pt_get_guest()
665 thread__zput(ptq->unknown_guest_thread); in intel_pt_get_guest()
671 ptq->unknown_guest_thread = machine__idle_thread(machine); in intel_pt_get_guest()
672 if (!ptq->unknown_guest_thread) in intel_pt_get_guest()
675 ptq->guest_machine = machine; in intel_pt_get_guest()
676 ptq->guest_machine_pid = pid; in intel_pt_get_guest()
686 struct intel_pt_queue *ptq = data; in intel_pt_walk_next_insn() local
687 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
704 nr = ptq->state->to_nr; in intel_pt_walk_next_insn()
705 cpumode = intel_pt_nr_cpumode(ptq, *ip, nr); in intel_pt_walk_next_insn()
709 intel_pt_get_guest(ptq)) in intel_pt_walk_next_insn()
711 machine = ptq->guest_machine; in intel_pt_walk_next_insn()
712 thread = ptq->unknown_guest_thread; in intel_pt_walk_next_insn()
714 thread = ptq->thread; in intel_pt_walk_next_insn()
718 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
863 struct intel_pt_queue *ptq = data; in __intel_pt_pgd_ip() local
869 if (ptq->state->to_nr) { in __intel_pt_pgd_ip()
871 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
874 } else if (ip >= ptq->pt->kernel_start) { in __intel_pt_pgd_ip()
875 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
880 thread = ptq->thread; in __intel_pt_pgd_ip()
889 return intel_pt_match_pgd_ip(ptq->pt, ip, offset, in __intel_pt_pgd_ip()
1156 struct intel_pt_queue *ptq; in intel_pt_alloc_queue() local
1158 ptq = zalloc(sizeof(struct intel_pt_queue)); in intel_pt_alloc_queue()
1159 if (!ptq) in intel_pt_alloc_queue()
1163 ptq->chain = intel_pt_alloc_chain(pt); in intel_pt_alloc_queue()
1164 if (!ptq->chain) in intel_pt_alloc_queue()
1171 ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); in intel_pt_alloc_queue()
1172 if (!ptq->last_branch) in intel_pt_alloc_queue()
1176 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); in intel_pt_alloc_queue()
1177 if (!ptq->event_buf) in intel_pt_alloc_queue()
1180 ptq->pt = pt; in intel_pt_alloc_queue()
1181 ptq->queue_nr = queue_nr; in intel_pt_alloc_queue()
1182 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
1183 ptq->pid = -1; in intel_pt_alloc_queue()
1184 ptq->tid = -1; in intel_pt_alloc_queue()
1185 ptq->cpu = -1; in intel_pt_alloc_queue()
1186 ptq->next_tid = -1; in intel_pt_alloc_queue()
1192 params.data = ptq; in intel_pt_alloc_queue()
1240 ptq->decoder = intel_pt_decoder_new(¶ms); in intel_pt_alloc_queue()
1241 if (!ptq->decoder) in intel_pt_alloc_queue()
1244 return ptq; in intel_pt_alloc_queue()
1247 zfree(&ptq->event_buf); in intel_pt_alloc_queue()
1248 zfree(&ptq->last_branch); in intel_pt_alloc_queue()
1249 zfree(&ptq->chain); in intel_pt_alloc_queue()
1250 free(ptq); in intel_pt_alloc_queue()
1256 struct intel_pt_queue *ptq = priv; in intel_pt_free_queue() local
1258 if (!ptq) in intel_pt_free_queue()
1260 thread__zput(ptq->thread); in intel_pt_free_queue()
1261 thread__zput(ptq->unknown_guest_thread); in intel_pt_free_queue()
1262 intel_pt_decoder_free(ptq->decoder); in intel_pt_free_queue()
1263 zfree(&ptq->event_buf); in intel_pt_free_queue()
1264 zfree(&ptq->last_branch); in intel_pt_free_queue()
1265 zfree(&ptq->chain); in intel_pt_free_queue()
1266 free(ptq); in intel_pt_free_queue()
1277 struct intel_pt_queue *ptq = queue->priv; in intel_pt_first_timestamp() local
1279 if (ptq && ptq->decoder) in intel_pt_first_timestamp()
1280 intel_pt_set_first_timestamp(ptq->decoder, timestamp); in intel_pt_first_timestamp()
1287 struct intel_pt_queue *ptq = queue->priv; in intel_pt_set_pid_tid_cpu() local
1290 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
1291 if (ptq->tid == -1) in intel_pt_set_pid_tid_cpu()
1292 ptq->pid = -1; in intel_pt_set_pid_tid_cpu()
1293 thread__zput(ptq->thread); in intel_pt_set_pid_tid_cpu()
1296 if (!ptq->thread && ptq->tid != -1) in intel_pt_set_pid_tid_cpu()
1297 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
1299 if (ptq->thread) { in intel_pt_set_pid_tid_cpu()
1300 ptq->pid = ptq->thread->pid_; in intel_pt_set_pid_tid_cpu()
1302 ptq->cpu = ptq->thread->cpu; in intel_pt_set_pid_tid_cpu()
1306 static void intel_pt_sample_flags(struct intel_pt_queue *ptq) in intel_pt_sample_flags() argument
1308 ptq->insn_len = 0; in intel_pt_sample_flags()
1309 if (ptq->state->flags & INTEL_PT_ABORT_TX) { in intel_pt_sample_flags()
1310 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT; in intel_pt_sample_flags()
1311 } else if (ptq->state->flags & INTEL_PT_ASYNC) { in intel_pt_sample_flags()
1312 if (!ptq->state->to_ip) in intel_pt_sample_flags()
1313 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1315 else if (ptq->state->from_nr && !ptq->state->to_nr) in intel_pt_sample_flags()
1316 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
1319 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
1323 if (ptq->state->from_ip) in intel_pt_sample_flags()
1324 ptq->flags = intel_pt_insn_type(ptq->state->insn_op); in intel_pt_sample_flags()
1326 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1328 if (ptq->state->flags & INTEL_PT_IN_TX) in intel_pt_sample_flags()
1329 ptq->flags |= PERF_IP_FLAG_IN_TX; in intel_pt_sample_flags()
1330 ptq->insn_len = ptq->state->insn_len; in intel_pt_sample_flags()
1331 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_sample_flags()
1334 if (ptq->state->type & INTEL_PT_TRACE_BEGIN) in intel_pt_sample_flags()
1335 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN; in intel_pt_sample_flags()
1336 if (ptq->state->type & INTEL_PT_TRACE_END) in intel_pt_sample_flags()
1337 ptq->flags |= PERF_IP_FLAG_TRACE_END; in intel_pt_sample_flags()
1341 struct intel_pt_queue *ptq) in intel_pt_setup_time_range() argument
1346 ptq->sel_timestamp = pt->time_ranges[0].start; in intel_pt_setup_time_range()
1347 ptq->sel_idx = 0; in intel_pt_setup_time_range()
1349 if (ptq->sel_timestamp) { in intel_pt_setup_time_range()
1350 ptq->sel_start = true; in intel_pt_setup_time_range()
1352 ptq->sel_timestamp = pt->time_ranges[0].end; in intel_pt_setup_time_range()
1353 ptq->sel_start = false; in intel_pt_setup_time_range()
1361 struct intel_pt_queue *ptq = queue->priv; in intel_pt_setup_queue() local
1366 if (!ptq) { in intel_pt_setup_queue()
1367 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
1368 if (!ptq) in intel_pt_setup_queue()
1370 queue->priv = ptq; in intel_pt_setup_queue()
1373 ptq->cpu = queue->cpu; in intel_pt_setup_queue()
1374 ptq->tid = queue->tid; in intel_pt_setup_queue()
1376 ptq->cbr_seen = UINT_MAX; in intel_pt_setup_queue()
1380 ptq->step_through_buffers = true; in intel_pt_setup_queue()
1382 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
1384 intel_pt_setup_time_range(pt, ptq); in intel_pt_setup_queue()
1387 if (!ptq->on_heap && in intel_pt_setup_queue()
1388 (!ptq->sync_switch || in intel_pt_setup_queue()
1389 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) { in intel_pt_setup_queue()
1398 queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_setup_queue()
1400 if (ptq->sel_start && ptq->sel_timestamp) { in intel_pt_setup_queue()
1401 ret = intel_pt_fast_forward(ptq->decoder, in intel_pt_setup_queue()
1402 ptq->sel_timestamp); in intel_pt_setup_queue()
1408 state = intel_pt_decode(ptq->decoder); in intel_pt_setup_queue()
1421 ptq->timestamp = state->timestamp; in intel_pt_setup_queue()
1423 queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1424 ptq->state = state; in intel_pt_setup_queue()
1425 ptq->have_sample = true; in intel_pt_setup_queue()
1426 if (ptq->sel_start && ptq->sel_timestamp && in intel_pt_setup_queue()
1427 ptq->timestamp < ptq->sel_timestamp) in intel_pt_setup_queue()
1428 ptq->have_sample = false; in intel_pt_setup_queue()
1429 intel_pt_sample_flags(ptq); in intel_pt_setup_queue()
1430 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1433 ptq->on_heap = true; in intel_pt_setup_queue()
1469 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq, in intel_pt_prep_a_sample() argument
1476 sample->pid = ptq->pid; in intel_pt_prep_a_sample()
1477 sample->tid = ptq->tid; in intel_pt_prep_a_sample()
1478 sample->cpu = ptq->cpu; in intel_pt_prep_a_sample()
1479 sample->insn_len = ptq->insn_len; in intel_pt_prep_a_sample()
1480 memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_prep_a_sample()
1484 struct intel_pt_queue *ptq, in intel_pt_prep_b_sample() argument
1488 intel_pt_prep_a_sample(ptq, event, sample); in intel_pt_prep_b_sample()
1491 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1493 sample->ip = ptq->state->from_ip; in intel_pt_prep_b_sample()
1494 sample->addr = ptq->state->to_ip; in intel_pt_prep_b_sample()
1495 sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr); in intel_pt_prep_b_sample()
1497 sample->flags = ptq->flags; in intel_pt_prep_b_sample()
1536 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq) in intel_pt_synth_branch_sample() argument
1538 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample()
1539 union perf_event *event = ptq->event_buf; in intel_pt_synth_branch_sample()
1547 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1553 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1555 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1556 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1574 if (ptq->state->flags & INTEL_PT_SAMPLE_IPC) in intel_pt_synth_branch_sample()
1575 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt; in intel_pt_synth_branch_sample()
1577 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt; in intel_pt_synth_branch_sample()
1578 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_branch_sample()
1579 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_branch_sample()
1587 struct intel_pt_queue *ptq, in intel_pt_prep_sample() argument
1591 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1594 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_prep_sample()
1597 sample->callchain = ptq->chain; in intel_pt_prep_sample()
1601 thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch, in intel_pt_prep_sample()
1603 sample->branch_stack = ptq->last_branch; in intel_pt_prep_sample()
1607 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_instruction_sample() argument
1609 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample()
1610 union perf_event *event = ptq->event_buf; in intel_pt_synth_instruction_sample()
1616 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1618 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1619 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1623 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt; in intel_pt_synth_instruction_sample()
1625 if (ptq->state->flags & INTEL_PT_SAMPLE_IPC) in intel_pt_synth_instruction_sample()
1626 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt; in intel_pt_synth_instruction_sample()
1628 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt; in intel_pt_synth_instruction_sample()
1629 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_instruction_sample()
1630 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_instruction_sample()
1633 ptq->last_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_synth_instruction_sample()
1639 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_transaction_sample() argument
1641 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample()
1642 union perf_event *event = ptq->event_buf; in intel_pt_synth_transaction_sample()
1648 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1650 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1651 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1658 struct intel_pt_queue *ptq, in intel_pt_prep_p_sample() argument
1662 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1672 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq) in intel_pt_synth_ptwrite_sample() argument
1674 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample()
1675 union perf_event *event = ptq->event_buf; in intel_pt_synth_ptwrite_sample()
1682 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1684 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1685 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1688 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_ptwrite_sample()
1689 raw.payload = cpu_to_le64(ptq->state->ptw_payload); in intel_pt_synth_ptwrite_sample()
1698 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq) in intel_pt_synth_cbr_sample() argument
1700 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample()
1701 union perf_event *event = ptq->event_buf; in intel_pt_synth_cbr_sample()
1709 ptq->cbr_seen = ptq->state->cbr; in intel_pt_synth_cbr_sample()
1711 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1713 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1714 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1716 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1728 static int intel_pt_synth_psb_sample(struct intel_pt_queue *ptq) in intel_pt_synth_psb_sample() argument
1730 struct intel_pt *pt = ptq->pt; in intel_pt_synth_psb_sample()
1731 union perf_event *event = ptq->event_buf; in intel_pt_synth_psb_sample()
1738 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_psb_sample()
1740 sample.id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1741 sample.stream_id = ptq->pt->psb_id; in intel_pt_synth_psb_sample()
1745 raw.offset = ptq->state->psb_offset; in intel_pt_synth_psb_sample()
1754 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq) in intel_pt_synth_mwait_sample() argument
1756 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample()
1757 union perf_event *event = ptq->event_buf; in intel_pt_synth_mwait_sample()
1764 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
1766 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1767 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1770 raw.payload = cpu_to_le64(ptq->state->mwait_payload); in intel_pt_synth_mwait_sample()
1779 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwre_sample() argument
1781 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample()
1782 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwre_sample()
1789 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
1791 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1792 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1795 raw.payload = cpu_to_le64(ptq->state->pwre_payload); in intel_pt_synth_pwre_sample()
1804 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq) in intel_pt_synth_exstop_sample() argument
1806 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample()
1807 union perf_event *event = ptq->event_buf; in intel_pt_synth_exstop_sample()
1814 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
1816 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1817 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1820 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_exstop_sample()
1829 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwrx_sample() argument
1831 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample()
1832 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwrx_sample()
1839 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
1841 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1842 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1845 raw.payload = cpu_to_le64(ptq->state->pwrx_payload); in intel_pt_synth_pwrx_sample()
1981 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pebs_sample() argument
1983 const struct intel_pt_blk_items *items = &ptq->state->items; in intel_pt_synth_pebs_sample()
1985 union perf_event *event = ptq->event_buf; in intel_pt_synth_pebs_sample()
1986 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pebs_sample()
1996 intel_pt_prep_a_sample(ptq, event, &sample); in intel_pt_synth_pebs_sample()
2010 sample.ip = ptq->state->from_ip; in intel_pt_synth_pebs_sample()
2012 cpumode = intel_pt_cpumode(ptq, sample.ip, 0); in intel_pt_synth_pebs_sample()
2024 timestamp = ptq->timestamp; in intel_pt_synth_pebs_sample()
2031 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_synth_pebs_sample()
2034 sample.callchain = ptq->chain; in intel_pt_synth_pebs_sample()
2057 intel_pt_add_lbrs(ptq->last_branch, items); in intel_pt_synth_pebs_sample()
2059 thread_stack__br_sample(ptq->thread, ptq->cpu, in intel_pt_synth_pebs_sample()
2060 ptq->last_branch, in intel_pt_synth_pebs_sample()
2063 ptq->last_branch->nr = 0; in intel_pt_synth_pebs_sample()
2065 sample.branch_stack = ptq->last_branch; in intel_pt_synth_pebs_sample()
2144 static int intel_ptq_synth_error(struct intel_pt_queue *ptq, in intel_ptq_synth_error() argument
2147 struct intel_pt *pt = ptq->pt; in intel_ptq_synth_error()
2148 u64 tm = ptq->timestamp; in intel_ptq_synth_error()
2152 return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid, in intel_ptq_synth_error()
2153 ptq->tid, state->from_ip, tm); in intel_ptq_synth_error()
2156 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
2159 pid_t tid = ptq->next_tid; in intel_pt_next_tid()
2165 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid); in intel_pt_next_tid()
2167 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
2169 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
2172 ptq->next_tid = -1; in intel_pt_next_tid()
2177 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip) in intel_pt_is_switch_ip() argument
2179 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip()
2182 (ptq->flags & PERF_IP_FLAG_BRANCH) && in intel_pt_is_switch_ip()
2183 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC | in intel_pt_is_switch_ip()
2190 static int intel_pt_sample(struct intel_pt_queue *ptq) in intel_pt_sample() argument
2192 const struct intel_pt_state *state = ptq->state; in intel_pt_sample()
2193 struct intel_pt *pt = ptq->pt; in intel_pt_sample()
2196 if (!ptq->have_sample) in intel_pt_sample()
2199 ptq->have_sample = false; in intel_pt_sample()
2201 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_sample()
2202 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt; in intel_pt_sample()
2209 err = intel_pt_synth_pebs_sample(ptq); in intel_pt_sample()
2216 err = intel_pt_synth_psb_sample(ptq); in intel_pt_sample()
2220 if (ptq->state->cbr != ptq->cbr_seen) { in intel_pt_sample()
2221 err = intel_pt_synth_cbr_sample(ptq); in intel_pt_sample()
2227 err = intel_pt_synth_mwait_sample(ptq); in intel_pt_sample()
2232 err = intel_pt_synth_pwre_sample(ptq); in intel_pt_sample()
2237 err = intel_pt_synth_exstop_sample(ptq); in intel_pt_sample()
2242 err = intel_pt_synth_pwrx_sample(ptq); in intel_pt_sample()
2250 err = intel_pt_synth_instruction_sample(ptq); in intel_pt_sample()
2256 err = intel_pt_synth_transaction_sample(ptq); in intel_pt_sample()
2262 err = intel_pt_synth_ptwrite_sample(ptq); in intel_pt_sample()
2271 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, in intel_pt_sample()
2272 state->from_ip, state->to_ip, ptq->insn_len, in intel_pt_sample()
2277 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr); in intel_pt_sample()
2292 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2297 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2300 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2306 if (!ptq->sync_switch) in intel_pt_sample()
2309 if (intel_pt_is_switch_ip(ptq, state->to_ip)) { in intel_pt_sample()
2310 switch (ptq->switch_state) { in intel_pt_sample()
2314 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
2317 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2320 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT; in intel_pt_sample()
2324 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_sample()
2325 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) { in intel_pt_sample()
2326 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_sample()
2327 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_sample()
2329 (ptq->flags & PERF_IP_FLAG_CALL)) { in intel_pt_sample()
2330 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2396 struct intel_pt_queue *ptq = queue->priv; in intel_pt_enable_sync_switch() local
2398 if (ptq) in intel_pt_enable_sync_switch()
2399 ptq->sync_switch = true; in intel_pt_enable_sync_switch()
2407 static bool intel_pt_next_time(struct intel_pt_queue *ptq) in intel_pt_next_time() argument
2409 struct intel_pt *pt = ptq->pt; in intel_pt_next_time()
2411 if (ptq->sel_start) { in intel_pt_next_time()
2413 ptq->sel_start = false; in intel_pt_next_time()
2414 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end; in intel_pt_next_time()
2416 } else if (ptq->sel_idx + 1 < pt->range_cnt) { in intel_pt_next_time()
2418 ptq->sel_start = true; in intel_pt_next_time()
2419 ptq->sel_idx += 1; in intel_pt_next_time()
2420 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start; in intel_pt_next_time()
2428 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp) in intel_pt_time_filter() argument
2433 if (ptq->sel_start) { in intel_pt_time_filter()
2434 if (ptq->timestamp >= ptq->sel_timestamp) { in intel_pt_time_filter()
2436 intel_pt_next_time(ptq); in intel_pt_time_filter()
2437 if (!ptq->sel_timestamp) { in intel_pt_time_filter()
2445 ptq->have_sample = false; in intel_pt_time_filter()
2446 if (ptq->sel_timestamp > *ff_timestamp) { in intel_pt_time_filter()
2447 if (ptq->sync_switch) { in intel_pt_time_filter()
2448 intel_pt_next_tid(ptq->pt, ptq); in intel_pt_time_filter()
2449 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_time_filter()
2451 *ff_timestamp = ptq->sel_timestamp; in intel_pt_time_filter()
2452 err = intel_pt_fast_forward(ptq->decoder, in intel_pt_time_filter()
2453 ptq->sel_timestamp); in intel_pt_time_filter()
2458 } else if (ptq->timestamp > ptq->sel_timestamp) { in intel_pt_time_filter()
2460 if (!intel_pt_next_time(ptq)) { in intel_pt_time_filter()
2462 ptq->have_sample = false; in intel_pt_time_filter()
2463 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_time_filter()
2475 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp) in intel_pt_run_decoder() argument
2477 const struct intel_pt_state *state = ptq->state; in intel_pt_run_decoder()
2478 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder()
2498 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_run_decoder()
2500 err = intel_pt_sample(ptq); in intel_pt_run_decoder()
2504 state = intel_pt_decode(ptq->decoder); in intel_pt_run_decoder()
2508 if (ptq->sync_switch && in intel_pt_run_decoder()
2510 ptq->sync_switch = false; in intel_pt_run_decoder()
2511 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
2514 err = intel_ptq_synth_error(ptq, state); in intel_pt_run_decoder()
2521 ptq->state = state; in intel_pt_run_decoder()
2522 ptq->have_sample = true; in intel_pt_run_decoder()
2523 intel_pt_sample_flags(ptq); in intel_pt_run_decoder()
2531 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2533 } else if (ptq->sync_switch && in intel_pt_run_decoder()
2534 ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_run_decoder()
2535 intel_pt_is_switch_ip(ptq, state->to_ip) && in intel_pt_run_decoder()
2536 ptq->next_tid == -1) { in intel_pt_run_decoder()
2539 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2540 } else if (state->timestamp > ptq->timestamp) { in intel_pt_run_decoder()
2541 ptq->timestamp = state->timestamp; in intel_pt_run_decoder()
2544 if (ptq->sel_timestamp) { in intel_pt_run_decoder()
2545 err = intel_pt_time_filter(ptq, &ff_timestamp); in intel_pt_run_decoder()
2550 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
2551 *timestamp = ptq->timestamp; in intel_pt_run_decoder()
2575 struct intel_pt_queue *ptq; in intel_pt_process_queues() local
2585 ptq = queue->priv; in intel_pt_process_queues()
2603 ret = intel_pt_run_decoder(ptq, &ts); in intel_pt_process_queues()
2615 ptq->on_heap = false; in intel_pt_process_queues()
2631 struct intel_pt_queue *ptq = queue->priv; in intel_pt_process_timeless_queues() local
2633 if (ptq && (tid == -1 || ptq->tid == tid)) { in intel_pt_process_timeless_queues()
2634 ptq->time = time_; in intel_pt_process_timeless_queues()
2636 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_queues()
2642 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq, in intel_pt_sample_set_pid_tid_cpu() argument
2646 struct machine *m = ptq->pt->machine; in intel_pt_sample_set_pid_tid_cpu()
2648 ptq->pid = sample->pid; in intel_pt_sample_set_pid_tid_cpu()
2649 ptq->tid = sample->tid; in intel_pt_sample_set_pid_tid_cpu()
2650 ptq->cpu = queue->cpu; in intel_pt_sample_set_pid_tid_cpu()
2653 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2655 thread__zput(ptq->thread); in intel_pt_sample_set_pid_tid_cpu()
2657 if (ptq->tid == -1) in intel_pt_sample_set_pid_tid_cpu()
2660 if (ptq->pid == -1) { in intel_pt_sample_set_pid_tid_cpu()
2661 ptq->thread = machine__find_thread(m, -1, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2662 if (ptq->thread) in intel_pt_sample_set_pid_tid_cpu()
2663 ptq->pid = ptq->thread->pid_; in intel_pt_sample_set_pid_tid_cpu()
2667 ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2674 struct intel_pt_queue *ptq; in intel_pt_process_timeless_sample() local
2681 ptq = queue->priv; in intel_pt_process_timeless_sample()
2682 if (!ptq) in intel_pt_process_timeless_sample()
2685 ptq->stop = false; in intel_pt_process_timeless_sample()
2686 ptq->time = sample->time; in intel_pt_process_timeless_sample()
2687 intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample); in intel_pt_process_timeless_sample()
2688 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_sample()
2729 struct intel_pt_queue *ptq; in intel_pt_sync_switch() local
2735 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
2736 if (!ptq || !ptq->sync_switch) in intel_pt_sync_switch()
2739 switch (ptq->switch_state) { in intel_pt_sync_switch()
2744 ptq->next_tid = tid; in intel_pt_sync_switch()
2745 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP; in intel_pt_sync_switch()
2748 if (!ptq->on_heap) { in intel_pt_sync_switch()
2749 ptq->timestamp = perf_time_to_tsc(timestamp, in intel_pt_sync_switch()
2751 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
2752 ptq->timestamp); in intel_pt_sync_switch()
2755 ptq->on_heap = true; in intel_pt_sync_switch()
2757 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sync_switch()
2766 ptq->next_tid = -1; in intel_pt_sync_switch()
2803 struct intel_pt_queue *ptq; in intel_pt_context_switch_in() local
2805 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_context_switch_in()
2806 if (ptq && ptq->sync_switch) { in intel_pt_context_switch_in()
2807 ptq->next_tid = -1; in intel_pt_context_switch_in()
2808 switch (ptq->switch_state) { in intel_pt_context_switch_in()
2815 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_context_switch_in()