Lines Matching refs:ptq
287 static int intel_pt_get_buffer(struct intel_pt_queue *ptq, in intel_pt_get_buffer() argument
295 int fd = perf_data__fd(ptq->pt->session->data); in intel_pt_get_buffer()
302 might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode; in intel_pt_get_buffer()
304 intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer)) in intel_pt_get_buffer()
327 static void intel_pt_lookahead_drop_buffer(struct intel_pt_queue *ptq, in intel_pt_lookahead_drop_buffer() argument
330 if (!buffer || buffer == ptq->buffer || buffer == ptq->old_buffer) in intel_pt_lookahead_drop_buffer()
340 struct intel_pt_queue *ptq = data; in intel_pt_lookahead() local
341 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_lookahead()
342 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_lookahead()
346 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_lookahead()
355 err = intel_pt_get_buffer(ptq, buffer, old_buffer, &b); in intel_pt_lookahead()
360 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
363 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
373 intel_pt_lookahead_drop_buffer(ptq, buffer); in intel_pt_lookahead()
374 intel_pt_lookahead_drop_buffer(ptq, old_buffer); in intel_pt_lookahead()
385 struct intel_pt_queue *ptq = data; in intel_pt_get_trace() local
386 struct auxtrace_buffer *buffer = ptq->buffer; in intel_pt_get_trace()
387 struct auxtrace_buffer *old_buffer = ptq->old_buffer; in intel_pt_get_trace()
391 if (ptq->stop) { in intel_pt_get_trace()
396 queue = &ptq->pt->queues.queue_array[ptq->queue_nr]; in intel_pt_get_trace()
406 ptq->buffer = buffer; in intel_pt_get_trace()
408 err = intel_pt_get_buffer(ptq, buffer, old_buffer, b); in intel_pt_get_trace()
412 if (ptq->step_through_buffers) in intel_pt_get_trace()
413 ptq->stop = true; in intel_pt_get_trace()
418 ptq->old_buffer = buffer; in intel_pt_get_trace()
564 struct intel_pt_queue *ptq = data; in intel_pt_walk_next_insn() local
565 struct machine *machine = ptq->pt->machine; in intel_pt_walk_next_insn()
581 cpumode = intel_pt_cpumode(ptq->pt, *ip); in intel_pt_walk_next_insn()
583 thread = ptq->thread; in intel_pt_walk_next_insn()
587 thread = ptq->pt->unknown_thread; in intel_pt_walk_next_insn()
729 struct intel_pt_queue *ptq = data; in __intel_pt_pgd_ip() local
735 if (ip >= ptq->pt->kernel_start) in __intel_pt_pgd_ip()
736 return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL); in __intel_pt_pgd_ip()
740 thread = ptq->thread; in __intel_pt_pgd_ip()
749 return intel_pt_match_pgd_ip(ptq->pt, ip, offset, in __intel_pt_pgd_ip()
991 struct intel_pt_queue *ptq; in intel_pt_alloc_queue() local
993 ptq = zalloc(sizeof(struct intel_pt_queue)); in intel_pt_alloc_queue()
994 if (!ptq) in intel_pt_alloc_queue()
998 ptq->chain = intel_pt_alloc_chain(pt); in intel_pt_alloc_queue()
999 if (!ptq->chain) in intel_pt_alloc_queue()
1006 ptq->last_branch = intel_pt_alloc_br_stack(entry_cnt); in intel_pt_alloc_queue()
1007 if (!ptq->last_branch) in intel_pt_alloc_queue()
1011 ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); in intel_pt_alloc_queue()
1012 if (!ptq->event_buf) in intel_pt_alloc_queue()
1015 ptq->pt = pt; in intel_pt_alloc_queue()
1016 ptq->queue_nr = queue_nr; in intel_pt_alloc_queue()
1017 ptq->exclude_kernel = intel_pt_exclude_kernel(pt); in intel_pt_alloc_queue()
1018 ptq->pid = -1; in intel_pt_alloc_queue()
1019 ptq->tid = -1; in intel_pt_alloc_queue()
1020 ptq->cpu = -1; in intel_pt_alloc_queue()
1021 ptq->next_tid = -1; in intel_pt_alloc_queue()
1026 params.data = ptq; in intel_pt_alloc_queue()
1069 ptq->decoder = intel_pt_decoder_new(¶ms); in intel_pt_alloc_queue()
1070 if (!ptq->decoder) in intel_pt_alloc_queue()
1073 return ptq; in intel_pt_alloc_queue()
1076 zfree(&ptq->event_buf); in intel_pt_alloc_queue()
1077 zfree(&ptq->last_branch); in intel_pt_alloc_queue()
1078 zfree(&ptq->chain); in intel_pt_alloc_queue()
1079 free(ptq); in intel_pt_alloc_queue()
1085 struct intel_pt_queue *ptq = priv; in intel_pt_free_queue() local
1087 if (!ptq) in intel_pt_free_queue()
1089 thread__zput(ptq->thread); in intel_pt_free_queue()
1090 intel_pt_decoder_free(ptq->decoder); in intel_pt_free_queue()
1091 zfree(&ptq->event_buf); in intel_pt_free_queue()
1092 zfree(&ptq->last_branch); in intel_pt_free_queue()
1093 zfree(&ptq->chain); in intel_pt_free_queue()
1094 free(ptq); in intel_pt_free_queue()
1100 struct intel_pt_queue *ptq = queue->priv; in intel_pt_set_pid_tid_cpu() local
1103 ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu); in intel_pt_set_pid_tid_cpu()
1104 if (ptq->tid == -1) in intel_pt_set_pid_tid_cpu()
1105 ptq->pid = -1; in intel_pt_set_pid_tid_cpu()
1106 thread__zput(ptq->thread); in intel_pt_set_pid_tid_cpu()
1109 if (!ptq->thread && ptq->tid != -1) in intel_pt_set_pid_tid_cpu()
1110 ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid); in intel_pt_set_pid_tid_cpu()
1112 if (ptq->thread) { in intel_pt_set_pid_tid_cpu()
1113 ptq->pid = ptq->thread->pid_; in intel_pt_set_pid_tid_cpu()
1115 ptq->cpu = ptq->thread->cpu; in intel_pt_set_pid_tid_cpu()
1119 static void intel_pt_sample_flags(struct intel_pt_queue *ptq) in intel_pt_sample_flags() argument
1121 if (ptq->state->flags & INTEL_PT_ABORT_TX) { in intel_pt_sample_flags()
1122 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT; in intel_pt_sample_flags()
1123 } else if (ptq->state->flags & INTEL_PT_ASYNC) { in intel_pt_sample_flags()
1124 if (ptq->state->to_ip) in intel_pt_sample_flags()
1125 ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | in intel_pt_sample_flags()
1129 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1131 ptq->insn_len = 0; in intel_pt_sample_flags()
1133 if (ptq->state->from_ip) in intel_pt_sample_flags()
1134 ptq->flags = intel_pt_insn_type(ptq->state->insn_op); in intel_pt_sample_flags()
1136 ptq->flags = PERF_IP_FLAG_BRANCH | in intel_pt_sample_flags()
1138 if (ptq->state->flags & INTEL_PT_IN_TX) in intel_pt_sample_flags()
1139 ptq->flags |= PERF_IP_FLAG_IN_TX; in intel_pt_sample_flags()
1140 ptq->insn_len = ptq->state->insn_len; in intel_pt_sample_flags()
1141 memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_sample_flags()
1144 if (ptq->state->type & INTEL_PT_TRACE_BEGIN) in intel_pt_sample_flags()
1145 ptq->flags |= PERF_IP_FLAG_TRACE_BEGIN; in intel_pt_sample_flags()
1146 if (ptq->state->type & INTEL_PT_TRACE_END) in intel_pt_sample_flags()
1147 ptq->flags |= PERF_IP_FLAG_TRACE_END; in intel_pt_sample_flags()
1151 struct intel_pt_queue *ptq) in intel_pt_setup_time_range() argument
1156 ptq->sel_timestamp = pt->time_ranges[0].start; in intel_pt_setup_time_range()
1157 ptq->sel_idx = 0; in intel_pt_setup_time_range()
1159 if (ptq->sel_timestamp) { in intel_pt_setup_time_range()
1160 ptq->sel_start = true; in intel_pt_setup_time_range()
1162 ptq->sel_timestamp = pt->time_ranges[0].end; in intel_pt_setup_time_range()
1163 ptq->sel_start = false; in intel_pt_setup_time_range()
1171 struct intel_pt_queue *ptq = queue->priv; in intel_pt_setup_queue() local
1176 if (!ptq) { in intel_pt_setup_queue()
1177 ptq = intel_pt_alloc_queue(pt, queue_nr); in intel_pt_setup_queue()
1178 if (!ptq) in intel_pt_setup_queue()
1180 queue->priv = ptq; in intel_pt_setup_queue()
1183 ptq->cpu = queue->cpu; in intel_pt_setup_queue()
1184 ptq->tid = queue->tid; in intel_pt_setup_queue()
1186 ptq->cbr_seen = UINT_MAX; in intel_pt_setup_queue()
1190 ptq->step_through_buffers = true; in intel_pt_setup_queue()
1192 ptq->sync_switch = pt->sync_switch; in intel_pt_setup_queue()
1194 intel_pt_setup_time_range(pt, ptq); in intel_pt_setup_queue()
1197 if (!ptq->on_heap && in intel_pt_setup_queue()
1198 (!ptq->sync_switch || in intel_pt_setup_queue()
1199 ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) { in intel_pt_setup_queue()
1208 queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_setup_queue()
1210 if (ptq->sel_start && ptq->sel_timestamp) { in intel_pt_setup_queue()
1211 ret = intel_pt_fast_forward(ptq->decoder, in intel_pt_setup_queue()
1212 ptq->sel_timestamp); in intel_pt_setup_queue()
1218 state = intel_pt_decode(ptq->decoder); in intel_pt_setup_queue()
1231 ptq->timestamp = state->timestamp; in intel_pt_setup_queue()
1233 queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1234 ptq->state = state; in intel_pt_setup_queue()
1235 ptq->have_sample = true; in intel_pt_setup_queue()
1236 if (ptq->sel_start && ptq->sel_timestamp && in intel_pt_setup_queue()
1237 ptq->timestamp < ptq->sel_timestamp) in intel_pt_setup_queue()
1238 ptq->have_sample = false; in intel_pt_setup_queue()
1239 intel_pt_sample_flags(ptq); in intel_pt_setup_queue()
1240 ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp); in intel_pt_setup_queue()
1243 ptq->on_heap = true; in intel_pt_setup_queue()
1279 static void intel_pt_prep_a_sample(struct intel_pt_queue *ptq, in intel_pt_prep_a_sample() argument
1286 sample->pid = ptq->pid; in intel_pt_prep_a_sample()
1287 sample->tid = ptq->tid; in intel_pt_prep_a_sample()
1288 sample->cpu = ptq->cpu; in intel_pt_prep_a_sample()
1289 sample->insn_len = ptq->insn_len; in intel_pt_prep_a_sample()
1290 memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ); in intel_pt_prep_a_sample()
1294 struct intel_pt_queue *ptq, in intel_pt_prep_b_sample() argument
1298 intel_pt_prep_a_sample(ptq, event, sample); in intel_pt_prep_b_sample()
1301 sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc); in intel_pt_prep_b_sample()
1303 sample->ip = ptq->state->from_ip; in intel_pt_prep_b_sample()
1305 sample->addr = ptq->state->to_ip; in intel_pt_prep_b_sample()
1307 sample->flags = ptq->flags; in intel_pt_prep_b_sample()
1346 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq) in intel_pt_synth_branch_sample() argument
1348 struct intel_pt *pt = ptq->pt; in intel_pt_synth_branch_sample()
1349 union perf_event *event = ptq->event_buf; in intel_pt_synth_branch_sample()
1357 if (pt->branches_filter && !(pt->branches_filter & ptq->flags)) in intel_pt_synth_branch_sample()
1363 intel_pt_prep_b_sample(pt, ptq, event, &sample); in intel_pt_synth_branch_sample()
1365 sample.id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1366 sample.stream_id = ptq->pt->branches_id; in intel_pt_synth_branch_sample()
1384 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt; in intel_pt_synth_branch_sample()
1386 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt; in intel_pt_synth_branch_sample()
1387 ptq->last_br_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_branch_sample()
1388 ptq->last_br_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_branch_sample()
1396 struct intel_pt_queue *ptq, in intel_pt_prep_sample() argument
1400 intel_pt_prep_b_sample(pt, ptq, event, sample); in intel_pt_prep_sample()
1403 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_prep_sample()
1406 sample->callchain = ptq->chain; in intel_pt_prep_sample()
1410 thread_stack__br_sample(ptq->thread, ptq->cpu, ptq->last_branch, in intel_pt_prep_sample()
1412 sample->branch_stack = ptq->last_branch; in intel_pt_prep_sample()
1416 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_instruction_sample() argument
1418 struct intel_pt *pt = ptq->pt; in intel_pt_synth_instruction_sample()
1419 union perf_event *event = ptq->event_buf; in intel_pt_synth_instruction_sample()
1425 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_instruction_sample()
1427 sample.id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1428 sample.stream_id = ptq->pt->instructions_id; in intel_pt_synth_instruction_sample()
1432 sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt; in intel_pt_synth_instruction_sample()
1434 sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt; in intel_pt_synth_instruction_sample()
1436 sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt; in intel_pt_synth_instruction_sample()
1437 ptq->last_in_insn_cnt = ptq->ipc_insn_cnt; in intel_pt_synth_instruction_sample()
1438 ptq->last_in_cyc_cnt = ptq->ipc_cyc_cnt; in intel_pt_synth_instruction_sample()
1441 ptq->last_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_synth_instruction_sample()
1447 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq) in intel_pt_synth_transaction_sample() argument
1449 struct intel_pt *pt = ptq->pt; in intel_pt_synth_transaction_sample()
1450 union perf_event *event = ptq->event_buf; in intel_pt_synth_transaction_sample()
1456 intel_pt_prep_sample(pt, ptq, event, &sample); in intel_pt_synth_transaction_sample()
1458 sample.id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1459 sample.stream_id = ptq->pt->transactions_id; in intel_pt_synth_transaction_sample()
1466 struct intel_pt_queue *ptq, in intel_pt_prep_p_sample() argument
1470 intel_pt_prep_sample(pt, ptq, event, sample); in intel_pt_prep_p_sample()
1480 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq) in intel_pt_synth_ptwrite_sample() argument
1482 struct intel_pt *pt = ptq->pt; in intel_pt_synth_ptwrite_sample()
1483 union perf_event *event = ptq->event_buf; in intel_pt_synth_ptwrite_sample()
1490 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_ptwrite_sample()
1492 sample.id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1493 sample.stream_id = ptq->pt->ptwrites_id; in intel_pt_synth_ptwrite_sample()
1496 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_ptwrite_sample()
1497 raw.payload = cpu_to_le64(ptq->state->ptw_payload); in intel_pt_synth_ptwrite_sample()
1506 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq) in intel_pt_synth_cbr_sample() argument
1508 struct intel_pt *pt = ptq->pt; in intel_pt_synth_cbr_sample()
1509 union perf_event *event = ptq->event_buf; in intel_pt_synth_cbr_sample()
1517 ptq->cbr_seen = ptq->state->cbr; in intel_pt_synth_cbr_sample()
1519 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_cbr_sample()
1521 sample.id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1522 sample.stream_id = ptq->pt->cbr_id; in intel_pt_synth_cbr_sample()
1524 flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16); in intel_pt_synth_cbr_sample()
1536 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq) in intel_pt_synth_mwait_sample() argument
1538 struct intel_pt *pt = ptq->pt; in intel_pt_synth_mwait_sample()
1539 union perf_event *event = ptq->event_buf; in intel_pt_synth_mwait_sample()
1546 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_mwait_sample()
1548 sample.id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1549 sample.stream_id = ptq->pt->mwait_id; in intel_pt_synth_mwait_sample()
1552 raw.payload = cpu_to_le64(ptq->state->mwait_payload); in intel_pt_synth_mwait_sample()
1561 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwre_sample() argument
1563 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwre_sample()
1564 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwre_sample()
1571 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwre_sample()
1573 sample.id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1574 sample.stream_id = ptq->pt->pwre_id; in intel_pt_synth_pwre_sample()
1577 raw.payload = cpu_to_le64(ptq->state->pwre_payload); in intel_pt_synth_pwre_sample()
1586 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq) in intel_pt_synth_exstop_sample() argument
1588 struct intel_pt *pt = ptq->pt; in intel_pt_synth_exstop_sample()
1589 union perf_event *event = ptq->event_buf; in intel_pt_synth_exstop_sample()
1596 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_exstop_sample()
1598 sample.id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1599 sample.stream_id = ptq->pt->exstop_id; in intel_pt_synth_exstop_sample()
1602 raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP); in intel_pt_synth_exstop_sample()
1611 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pwrx_sample() argument
1613 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pwrx_sample()
1614 union perf_event *event = ptq->event_buf; in intel_pt_synth_pwrx_sample()
1621 intel_pt_prep_p_sample(pt, ptq, event, &sample); in intel_pt_synth_pwrx_sample()
1623 sample.id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1624 sample.stream_id = ptq->pt->pwrx_id; in intel_pt_synth_pwrx_sample()
1627 raw.payload = cpu_to_le64(ptq->state->pwrx_payload); in intel_pt_synth_pwrx_sample()
1763 static int intel_pt_synth_pebs_sample(struct intel_pt_queue *ptq) in intel_pt_synth_pebs_sample() argument
1765 const struct intel_pt_blk_items *items = &ptq->state->items; in intel_pt_synth_pebs_sample()
1767 union perf_event *event = ptq->event_buf; in intel_pt_synth_pebs_sample()
1768 struct intel_pt *pt = ptq->pt; in intel_pt_synth_pebs_sample()
1778 intel_pt_prep_a_sample(ptq, event, &sample); in intel_pt_synth_pebs_sample()
1792 sample.ip = ptq->state->from_ip; in intel_pt_synth_pebs_sample()
1795 cpumode = sample.ip < ptq->pt->kernel_start ? in intel_pt_synth_pebs_sample()
1809 timestamp = ptq->timestamp; in intel_pt_synth_pebs_sample()
1816 thread_stack__sample(ptq->thread, ptq->cpu, ptq->chain, in intel_pt_synth_pebs_sample()
1819 sample.callchain = ptq->chain; in intel_pt_synth_pebs_sample()
1842 intel_pt_add_lbrs(ptq->last_branch, items); in intel_pt_synth_pebs_sample()
1844 thread_stack__br_sample(ptq->thread, ptq->cpu, in intel_pt_synth_pebs_sample()
1845 ptq->last_branch, in intel_pt_synth_pebs_sample()
1848 ptq->last_branch->nr = 0; in intel_pt_synth_pebs_sample()
1850 sample.branch_stack = ptq->last_branch; in intel_pt_synth_pebs_sample()
1912 static int intel_ptq_synth_error(struct intel_pt_queue *ptq, in intel_ptq_synth_error() argument
1915 struct intel_pt *pt = ptq->pt; in intel_ptq_synth_error()
1916 u64 tm = ptq->timestamp; in intel_ptq_synth_error()
1920 return intel_pt_synth_error(pt, state->err, ptq->cpu, ptq->pid, in intel_ptq_synth_error()
1921 ptq->tid, state->from_ip, tm); in intel_ptq_synth_error()
1924 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq) in intel_pt_next_tid() argument
1927 pid_t tid = ptq->next_tid; in intel_pt_next_tid()
1933 intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid); in intel_pt_next_tid()
1935 err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid); in intel_pt_next_tid()
1937 queue = &pt->queues.queue_array[ptq->queue_nr]; in intel_pt_next_tid()
1940 ptq->next_tid = -1; in intel_pt_next_tid()
1945 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip) in intel_pt_is_switch_ip() argument
1947 struct intel_pt *pt = ptq->pt; in intel_pt_is_switch_ip()
1950 (ptq->flags & PERF_IP_FLAG_BRANCH) && in intel_pt_is_switch_ip()
1951 !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC | in intel_pt_is_switch_ip()
1958 static int intel_pt_sample(struct intel_pt_queue *ptq) in intel_pt_sample() argument
1960 const struct intel_pt_state *state = ptq->state; in intel_pt_sample()
1961 struct intel_pt *pt = ptq->pt; in intel_pt_sample()
1964 if (!ptq->have_sample) in intel_pt_sample()
1967 ptq->have_sample = false; in intel_pt_sample()
1969 if (ptq->state->tot_cyc_cnt > ptq->ipc_cyc_cnt) { in intel_pt_sample()
1974 ptq->ipc_insn_cnt = ptq->state->tot_insn_cnt; in intel_pt_sample()
1975 ptq->ipc_cyc_cnt = ptq->state->tot_cyc_cnt; in intel_pt_sample()
1983 err = intel_pt_synth_pebs_sample(ptq); in intel_pt_sample()
1989 if (ptq->state->cbr != ptq->cbr_seen) { in intel_pt_sample()
1990 err = intel_pt_synth_cbr_sample(ptq); in intel_pt_sample()
1996 err = intel_pt_synth_mwait_sample(ptq); in intel_pt_sample()
2001 err = intel_pt_synth_pwre_sample(ptq); in intel_pt_sample()
2006 err = intel_pt_synth_exstop_sample(ptq); in intel_pt_sample()
2011 err = intel_pt_synth_pwrx_sample(ptq); in intel_pt_sample()
2019 err = intel_pt_synth_instruction_sample(ptq); in intel_pt_sample()
2025 err = intel_pt_synth_transaction_sample(ptq); in intel_pt_sample()
2031 err = intel_pt_synth_ptwrite_sample(ptq); in intel_pt_sample()
2040 thread_stack__event(ptq->thread, ptq->cpu, ptq->flags, in intel_pt_sample()
2041 state->from_ip, state->to_ip, ptq->insn_len, in intel_pt_sample()
2046 thread_stack__set_trace_nr(ptq->thread, ptq->cpu, state->trace_nr); in intel_pt_sample()
2050 err = intel_pt_synth_branch_sample(ptq); in intel_pt_sample()
2055 if (!ptq->sync_switch) in intel_pt_sample()
2058 if (intel_pt_is_switch_ip(ptq, state->to_ip)) { in intel_pt_sample()
2059 switch (ptq->switch_state) { in intel_pt_sample()
2063 err = intel_pt_next_tid(pt, ptq); in intel_pt_sample()
2066 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2069 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT; in intel_pt_sample()
2073 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_sample()
2074 } else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) { in intel_pt_sample()
2075 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_sample()
2076 } else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_sample()
2078 (ptq->flags & PERF_IP_FLAG_CALL)) { in intel_pt_sample()
2079 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sample()
2145 struct intel_pt_queue *ptq = queue->priv; in intel_pt_enable_sync_switch() local
2147 if (ptq) in intel_pt_enable_sync_switch()
2148 ptq->sync_switch = true; in intel_pt_enable_sync_switch()
2156 static bool intel_pt_next_time(struct intel_pt_queue *ptq) in intel_pt_next_time() argument
2158 struct intel_pt *pt = ptq->pt; in intel_pt_next_time()
2160 if (ptq->sel_start) { in intel_pt_next_time()
2162 ptq->sel_start = false; in intel_pt_next_time()
2163 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].end; in intel_pt_next_time()
2165 } else if (ptq->sel_idx + 1 < pt->range_cnt) { in intel_pt_next_time()
2167 ptq->sel_start = true; in intel_pt_next_time()
2168 ptq->sel_idx += 1; in intel_pt_next_time()
2169 ptq->sel_timestamp = pt->time_ranges[ptq->sel_idx].start; in intel_pt_next_time()
2177 static int intel_pt_time_filter(struct intel_pt_queue *ptq, u64 *ff_timestamp) in intel_pt_time_filter() argument
2182 if (ptq->sel_start) { in intel_pt_time_filter()
2183 if (ptq->timestamp >= ptq->sel_timestamp) { in intel_pt_time_filter()
2185 intel_pt_next_time(ptq); in intel_pt_time_filter()
2186 if (!ptq->sel_timestamp) { in intel_pt_time_filter()
2194 ptq->have_sample = false; in intel_pt_time_filter()
2195 if (ptq->sel_timestamp > *ff_timestamp) { in intel_pt_time_filter()
2196 if (ptq->sync_switch) { in intel_pt_time_filter()
2197 intel_pt_next_tid(ptq->pt, ptq); in intel_pt_time_filter()
2198 ptq->switch_state = INTEL_PT_SS_UNKNOWN; in intel_pt_time_filter()
2200 *ff_timestamp = ptq->sel_timestamp; in intel_pt_time_filter()
2201 err = intel_pt_fast_forward(ptq->decoder, in intel_pt_time_filter()
2202 ptq->sel_timestamp); in intel_pt_time_filter()
2207 } else if (ptq->timestamp > ptq->sel_timestamp) { in intel_pt_time_filter()
2209 if (!intel_pt_next_time(ptq)) { in intel_pt_time_filter()
2211 ptq->have_sample = false; in intel_pt_time_filter()
2212 ptq->switch_state = INTEL_PT_SS_NOT_TRACING; in intel_pt_time_filter()
2224 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp) in intel_pt_run_decoder() argument
2226 const struct intel_pt_state *state = ptq->state; in intel_pt_run_decoder()
2227 struct intel_pt *pt = ptq->pt; in intel_pt_run_decoder()
2247 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_run_decoder()
2249 err = intel_pt_sample(ptq); in intel_pt_run_decoder()
2253 state = intel_pt_decode(ptq->decoder); in intel_pt_run_decoder()
2257 if (ptq->sync_switch && in intel_pt_run_decoder()
2259 ptq->sync_switch = false; in intel_pt_run_decoder()
2260 intel_pt_next_tid(pt, ptq); in intel_pt_run_decoder()
2263 err = intel_ptq_synth_error(ptq, state); in intel_pt_run_decoder()
2270 ptq->state = state; in intel_pt_run_decoder()
2271 ptq->have_sample = true; in intel_pt_run_decoder()
2272 intel_pt_sample_flags(ptq); in intel_pt_run_decoder()
2280 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2282 } else if (ptq->sync_switch && in intel_pt_run_decoder()
2283 ptq->switch_state == INTEL_PT_SS_UNKNOWN && in intel_pt_run_decoder()
2284 intel_pt_is_switch_ip(ptq, state->to_ip) && in intel_pt_run_decoder()
2285 ptq->next_tid == -1) { in intel_pt_run_decoder()
2288 ptq->timestamp = state->est_timestamp; in intel_pt_run_decoder()
2289 } else if (state->timestamp > ptq->timestamp) { in intel_pt_run_decoder()
2290 ptq->timestamp = state->timestamp; in intel_pt_run_decoder()
2293 if (ptq->sel_timestamp) { in intel_pt_run_decoder()
2294 err = intel_pt_time_filter(ptq, &ff_timestamp); in intel_pt_run_decoder()
2299 if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) { in intel_pt_run_decoder()
2300 *timestamp = ptq->timestamp; in intel_pt_run_decoder()
2324 struct intel_pt_queue *ptq; in intel_pt_process_queues() local
2334 ptq = queue->priv; in intel_pt_process_queues()
2352 ret = intel_pt_run_decoder(ptq, &ts); in intel_pt_process_queues()
2364 ptq->on_heap = false; in intel_pt_process_queues()
2380 struct intel_pt_queue *ptq = queue->priv; in intel_pt_process_timeless_queues() local
2382 if (ptq && (tid == -1 || ptq->tid == tid)) { in intel_pt_process_timeless_queues()
2383 ptq->time = time_; in intel_pt_process_timeless_queues()
2385 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_queues()
2391 static void intel_pt_sample_set_pid_tid_cpu(struct intel_pt_queue *ptq, in intel_pt_sample_set_pid_tid_cpu() argument
2395 struct machine *m = ptq->pt->machine; in intel_pt_sample_set_pid_tid_cpu()
2397 ptq->pid = sample->pid; in intel_pt_sample_set_pid_tid_cpu()
2398 ptq->tid = sample->tid; in intel_pt_sample_set_pid_tid_cpu()
2399 ptq->cpu = queue->cpu; in intel_pt_sample_set_pid_tid_cpu()
2402 ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2404 thread__zput(ptq->thread); in intel_pt_sample_set_pid_tid_cpu()
2406 if (ptq->tid == -1) in intel_pt_sample_set_pid_tid_cpu()
2409 if (ptq->pid == -1) { in intel_pt_sample_set_pid_tid_cpu()
2410 ptq->thread = machine__find_thread(m, -1, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2411 if (ptq->thread) in intel_pt_sample_set_pid_tid_cpu()
2412 ptq->pid = ptq->thread->pid_; in intel_pt_sample_set_pid_tid_cpu()
2416 ptq->thread = machine__findnew_thread(m, ptq->pid, ptq->tid); in intel_pt_sample_set_pid_tid_cpu()
2423 struct intel_pt_queue *ptq; in intel_pt_process_timeless_sample() local
2430 ptq = queue->priv; in intel_pt_process_timeless_sample()
2431 if (!ptq) in intel_pt_process_timeless_sample()
2434 ptq->stop = false; in intel_pt_process_timeless_sample()
2435 ptq->time = sample->time; in intel_pt_process_timeless_sample()
2436 intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample); in intel_pt_process_timeless_sample()
2437 intel_pt_run_decoder(ptq, &ts); in intel_pt_process_timeless_sample()
2478 struct intel_pt_queue *ptq; in intel_pt_sync_switch() local
2484 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_sync_switch()
2485 if (!ptq || !ptq->sync_switch) in intel_pt_sync_switch()
2488 switch (ptq->switch_state) { in intel_pt_sync_switch()
2493 ptq->next_tid = tid; in intel_pt_sync_switch()
2494 ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP; in intel_pt_sync_switch()
2497 if (!ptq->on_heap) { in intel_pt_sync_switch()
2498 ptq->timestamp = perf_time_to_tsc(timestamp, in intel_pt_sync_switch()
2500 err = auxtrace_heap__add(&pt->heap, ptq->queue_nr, in intel_pt_sync_switch()
2501 ptq->timestamp); in intel_pt_sync_switch()
2504 ptq->on_heap = true; in intel_pt_sync_switch()
2506 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_sync_switch()
2515 ptq->next_tid = -1; in intel_pt_sync_switch()
2553 struct intel_pt_queue *ptq; in intel_pt_context_switch_in() local
2555 ptq = intel_pt_cpu_to_ptq(pt, cpu); in intel_pt_context_switch_in()
2556 if (ptq && ptq->sync_switch) { in intel_pt_context_switch_in()
2557 ptq->next_tid = -1; in intel_pt_context_switch_in()
2558 switch (ptq->switch_state) { in intel_pt_context_switch_in()
2565 ptq->switch_state = INTEL_PT_SS_TRACING; in intel_pt_context_switch_in()