229 	struct intel_pt_queue *ptq = data;  in intel_pt_get_trace()  local
230 	struct auxtrace_buffer *buffer = ptq->buffer;  in intel_pt_get_trace()
231 	struct auxtrace_buffer *old_buffer = ptq->old_buffer;  in intel_pt_get_trace()
235 	if (ptq->stop) {  in intel_pt_get_trace()
240 	queue = &ptq->pt->queues.queue_array[ptq->queue_nr];  in intel_pt_get_trace()
250 	ptq->buffer = buffer;  in intel_pt_get_trace()
253 		int fd = perf_data__fd(ptq->pt->session->data);  in intel_pt_get_trace()
260 	might_overlap = ptq->pt->snapshot_mode || ptq->pt->sampling_mode;  in intel_pt_get_trace()
262 	    intel_pt_do_fix_overlap(ptq->pt, old_buffer, buffer))  in intel_pt_get_trace()
281 	if (ptq->step_through_buffers)  in intel_pt_get_trace()
282 		ptq->stop = true;  in intel_pt_get_trace()
287 		ptq->old_buffer = buffer;  in intel_pt_get_trace()
415 	struct intel_pt_queue *ptq = data;  in intel_pt_walk_next_insn()  local
416 	struct machine *machine = ptq->pt->machine;  in intel_pt_walk_next_insn()
432 	if (*ip >= ptq->pt->kernel_start)  in intel_pt_walk_next_insn()
437 	thread = ptq->thread;  in intel_pt_walk_next_insn()
441 		thread = ptq->pt->unknown_thread;  in intel_pt_walk_next_insn()
583 	struct intel_pt_queue *ptq = data;  in __intel_pt_pgd_ip()  local
589 	if (ip >= ptq->pt->kernel_start)  in __intel_pt_pgd_ip()
590 		return intel_pt_match_pgd_ip(ptq->pt, ip, ip, NULL);  in __intel_pt_pgd_ip()
594 	thread = ptq->thread;  in __intel_pt_pgd_ip()
603 	return intel_pt_match_pgd_ip(ptq->pt, ip, offset,  in __intel_pt_pgd_ip()
753 	struct intel_pt_queue *ptq;  in intel_pt_alloc_queue()  local
755 	ptq = zalloc(sizeof(struct intel_pt_queue));  in intel_pt_alloc_queue()
756 	if (!ptq)  in intel_pt_alloc_queue()
763 		ptq->chain = zalloc(sz);  in intel_pt_alloc_queue()
764 		if (!ptq->chain)  in intel_pt_alloc_queue()
773 		ptq->last_branch = zalloc(sz);  in intel_pt_alloc_queue()
774 		if (!ptq->last_branch)  in intel_pt_alloc_queue()
776 		ptq->last_branch_rb = zalloc(sz);  in intel_pt_alloc_queue()
777 		if (!ptq->last_branch_rb)  in intel_pt_alloc_queue()
781 	ptq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in intel_pt_alloc_queue()
782 	if (!ptq->event_buf)  in intel_pt_alloc_queue()
785 	ptq->pt = pt;  in intel_pt_alloc_queue()
786 	ptq->queue_nr = queue_nr;  in intel_pt_alloc_queue()
787 	ptq->exclude_kernel = intel_pt_exclude_kernel(pt);  in intel_pt_alloc_queue()
788 	ptq->pid = -1;  in intel_pt_alloc_queue()
789 	ptq->tid = -1;  in intel_pt_alloc_queue()
790 	ptq->cpu = -1;  in intel_pt_alloc_queue()
791 	ptq->next_tid = -1;  in intel_pt_alloc_queue()
795 	params.data = ptq;  in intel_pt_alloc_queue()
837 	ptq->decoder = intel_pt_decoder_new(&params);  in intel_pt_alloc_queue()
838 	if (!ptq->decoder)  in intel_pt_alloc_queue()
841 	return ptq;  in intel_pt_alloc_queue()
844 	zfree(&ptq->event_buf);  in intel_pt_alloc_queue()
845 	zfree(&ptq->last_branch);  in intel_pt_alloc_queue()
846 	zfree(&ptq->last_branch_rb);  in intel_pt_alloc_queue()
847 	zfree(&ptq->chain);  in intel_pt_alloc_queue()
848 	free(ptq);  in intel_pt_alloc_queue()
854 	struct intel_pt_queue *ptq = priv;  in intel_pt_free_queue()  local
856 	if (!ptq)  in intel_pt_free_queue()
858 	thread__zput(ptq->thread);  in intel_pt_free_queue()
859 	intel_pt_decoder_free(ptq->decoder);  in intel_pt_free_queue()
860 	zfree(&ptq->event_buf);  in intel_pt_free_queue()
861 	zfree(&ptq->last_branch);  in intel_pt_free_queue()
862 	zfree(&ptq->last_branch_rb);  in intel_pt_free_queue()
863 	zfree(&ptq->chain);  in intel_pt_free_queue()
864 	free(ptq);  in intel_pt_free_queue()
870 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_set_pid_tid_cpu()  local
873 		ptq->tid = machine__get_current_tid(pt->machine, ptq->cpu);  in intel_pt_set_pid_tid_cpu()
874 		thread__zput(ptq->thread);  in intel_pt_set_pid_tid_cpu()
877 	if (!ptq->thread && ptq->tid != -1)  in intel_pt_set_pid_tid_cpu()
878 		ptq->thread = machine__find_thread(pt->machine, -1, ptq->tid);  in intel_pt_set_pid_tid_cpu()
880 	if (ptq->thread) {  in intel_pt_set_pid_tid_cpu()
881 		ptq->pid = ptq->thread->pid_;  in intel_pt_set_pid_tid_cpu()
883 			ptq->cpu = ptq->thread->cpu;  in intel_pt_set_pid_tid_cpu()
887 static void intel_pt_sample_flags(struct intel_pt_queue *ptq)  in intel_pt_sample_flags()  argument
889 	if (ptq->state->flags & INTEL_PT_ABORT_TX) {  in intel_pt_sample_flags()
890 		ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT;  in intel_pt_sample_flags()
891 	} else if (ptq->state->flags & INTEL_PT_ASYNC) {  in intel_pt_sample_flags()
892 		if (ptq->state->to_ip)  in intel_pt_sample_flags()
893 			ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL |  in intel_pt_sample_flags()
897 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
899 		ptq->insn_len = 0;  in intel_pt_sample_flags()
901 		if (ptq->state->from_ip)  in intel_pt_sample_flags()
902 			ptq->flags = intel_pt_insn_type(ptq->state->insn_op);  in intel_pt_sample_flags()
904 			ptq->flags = PERF_IP_FLAG_BRANCH |  in intel_pt_sample_flags()
906 		if (ptq->state->flags & INTEL_PT_IN_TX)  in intel_pt_sample_flags()
907 			ptq->flags |= PERF_IP_FLAG_IN_TX;  in intel_pt_sample_flags()
908 		ptq->insn_len = ptq->state->insn_len;  in intel_pt_sample_flags()
909 		memcpy(ptq->insn, ptq->state->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_sample_flags()
917 	struct intel_pt_queue *ptq = queue->priv;  in intel_pt_setup_queue()  local
922 	if (!ptq) {  in intel_pt_setup_queue()
923 		ptq = intel_pt_alloc_queue(pt, queue_nr);  in intel_pt_setup_queue()
924 		if (!ptq)  in intel_pt_setup_queue()
926 		queue->priv = ptq;  in intel_pt_setup_queue()
929 			ptq->cpu = queue->cpu;  in intel_pt_setup_queue()
930 		ptq->tid = queue->tid;  in intel_pt_setup_queue()
934 			ptq->step_through_buffers = true;  in intel_pt_setup_queue()
936 		ptq->sync_switch = pt->sync_switch;  in intel_pt_setup_queue()
939 	if (!ptq->on_heap &&  in intel_pt_setup_queue()
940 	    (!ptq->sync_switch ||  in intel_pt_setup_queue()
941 	     ptq->switch_state != INTEL_PT_SS_EXPECTING_SWITCH_EVENT)) {  in intel_pt_setup_queue()
950 			     queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_setup_queue()
952 			state = intel_pt_decode(ptq->decoder);  in intel_pt_setup_queue()
965 		ptq->timestamp = state->timestamp;  in intel_pt_setup_queue()
967 			     queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
968 		ptq->state = state;  in intel_pt_setup_queue()
969 		ptq->have_sample = true;  in intel_pt_setup_queue()
970 		intel_pt_sample_flags(ptq);  in intel_pt_setup_queue()
971 		ret = auxtrace_heap__add(&pt->heap, queue_nr, ptq->timestamp);  in intel_pt_setup_queue()
974 		ptq->on_heap = true;  in intel_pt_setup_queue()
993 static inline void intel_pt_copy_last_branch_rb(struct intel_pt_queue *ptq)  in intel_pt_copy_last_branch_rb()  argument
995 	struct branch_stack *bs_src = ptq->last_branch_rb;  in intel_pt_copy_last_branch_rb()
996 	struct branch_stack *bs_dst = ptq->last_branch;  in intel_pt_copy_last_branch_rb()
1004 	nr = ptq->pt->synth_opts.last_branch_sz - ptq->last_branch_pos;  in intel_pt_copy_last_branch_rb()
1006 	       &bs_src->entries[ptq->last_branch_pos],  in intel_pt_copy_last_branch_rb()
1009 	if (bs_src->nr >= ptq->pt->synth_opts.last_branch_sz) {  in intel_pt_copy_last_branch_rb()
1012 		       sizeof(struct branch_entry) * ptq->last_branch_pos);  in intel_pt_copy_last_branch_rb()
1016 static inline void intel_pt_reset_last_branch_rb(struct intel_pt_queue *ptq)  in intel_pt_reset_last_branch_rb()  argument
1018 	ptq->last_branch_pos = 0;  in intel_pt_reset_last_branch_rb()
1019 	ptq->last_branch_rb->nr = 0;  in intel_pt_reset_last_branch_rb()
1022 static void intel_pt_update_last_branch_rb(struct intel_pt_queue *ptq)  in intel_pt_update_last_branch_rb()  argument
1024 	const struct intel_pt_state *state = ptq->state;  in intel_pt_update_last_branch_rb()
1025 	struct branch_stack *bs = ptq->last_branch_rb;  in intel_pt_update_last_branch_rb()
1028 	if (!ptq->last_branch_pos)  in intel_pt_update_last_branch_rb()
1029 		ptq->last_branch_pos = ptq->pt->synth_opts.last_branch_sz;  in intel_pt_update_last_branch_rb()
1031 	ptq->last_branch_pos -= 1;  in intel_pt_update_last_branch_rb()
1033 	be              = &bs->entries[ptq->last_branch_pos];  in intel_pt_update_last_branch_rb()
1039 	be->flags.mispred = ptq->pt->mispred_all;  in intel_pt_update_last_branch_rb()
1041 	if (bs->nr < ptq->pt->synth_opts.last_branch_sz)  in intel_pt_update_last_branch_rb()
1052 				   struct intel_pt_queue *ptq,  in intel_pt_prep_b_sample()  argument
1061 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1064 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1065 	sample->pid = ptq->pid;  in intel_pt_prep_b_sample()
1066 	sample->tid = ptq->tid;  in intel_pt_prep_b_sample()
1067 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1069 	sample->cpu = ptq->cpu;  in intel_pt_prep_b_sample()
1070 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1071 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_b_sample()
1072 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_b_sample()
1109 static int intel_pt_synth_branch_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_branch_sample()  argument
1111 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_branch_sample()
1112 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_branch_sample()
1119 	if (pt->branches_filter && !(pt->branches_filter & ptq->flags))  in intel_pt_synth_branch_sample()
1125 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1127 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1128 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1150 				 struct intel_pt_queue *ptq,  in intel_pt_prep_sample()  argument
1154 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1157 		thread_stack__sample(ptq->thread, ptq->chain,  in intel_pt_prep_sample()
1159 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1163 		intel_pt_copy_last_branch_rb(ptq);  in intel_pt_prep_sample()
1164 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1169 					       struct intel_pt_queue *ptq,  in intel_pt_deliver_synth_event()  argument
1179 		intel_pt_reset_last_branch_rb(ptq);  in intel_pt_deliver_synth_event()
1184 static int intel_pt_synth_instruction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_instruction_sample()  argument
1186 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_instruction_sample()
1187 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_instruction_sample()
1193 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1195 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1196 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1197 	sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1199 	ptq->last_insn_cnt = ptq->state->tot_insn_cnt;  in intel_pt_synth_instruction_sample()
1201 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_instruction_sample()
1205 static int intel_pt_synth_transaction_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_transaction_sample()  argument
1207 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_transaction_sample()
1208 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_transaction_sample()
1214 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1216 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1217 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1219 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_transaction_sample()
1224 				   struct intel_pt_queue *ptq,  in intel_pt_prep_p_sample()  argument
1228 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1238 static int intel_pt_synth_ptwrite_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_ptwrite_sample()  argument
1240 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_ptwrite_sample()
1241 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_ptwrite_sample()
1248 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1250 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1251 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1254 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_ptwrite_sample()
1255 	raw.payload = cpu_to_le64(ptq->state->ptw_payload);  in intel_pt_synth_ptwrite_sample()
1260 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_ptwrite_sample()
1264 static int intel_pt_synth_cbr_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_cbr_sample()  argument
1266 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_cbr_sample()
1267 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_cbr_sample()
1275 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1277 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1278 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1280 	flags = (u16)ptq->state->cbr_payload | (pt->max_non_turbo_ratio << 16);  in intel_pt_synth_cbr_sample()
1288 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_cbr_sample()
1292 static int intel_pt_synth_mwait_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_mwait_sample()  argument
1294 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_mwait_sample()
1295 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_mwait_sample()
1302 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1304 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1305 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1308 	raw.payload = cpu_to_le64(ptq->state->mwait_payload);  in intel_pt_synth_mwait_sample()
1313 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_mwait_sample()
1317 static int intel_pt_synth_pwre_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwre_sample()  argument
1319 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwre_sample()
1320 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwre_sample()
1327 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1329 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1330 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1333 	raw.payload = cpu_to_le64(ptq->state->pwre_payload);  in intel_pt_synth_pwre_sample()
1338 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_pwre_sample()
1342 static int intel_pt_synth_exstop_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_exstop_sample()  argument
1344 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_exstop_sample()
1345 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_exstop_sample()
1352 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
1354 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1355 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1358 	raw.ip = !!(ptq->state->flags & INTEL_PT_FUP_IP);  in intel_pt_synth_exstop_sample()
1363 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_exstop_sample()
1367 static int intel_pt_synth_pwrx_sample(struct intel_pt_queue *ptq)  in intel_pt_synth_pwrx_sample()  argument
1369 	struct intel_pt *pt = ptq->pt;  in intel_pt_synth_pwrx_sample()
1370 	union perf_event *event = ptq->event_buf;  in intel_pt_synth_pwrx_sample()
1377 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
1379 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1380 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1383 	raw.payload = cpu_to_le64(ptq->state->pwrx_payload);  in intel_pt_synth_pwrx_sample()
1388 	return intel_pt_deliver_synth_event(pt, ptq, event, &sample,  in intel_pt_synth_pwrx_sample()
1412 static int intel_pt_next_tid(struct intel_pt *pt, struct intel_pt_queue *ptq)  in intel_pt_next_tid()  argument
1415 	pid_t tid = ptq->next_tid;  in intel_pt_next_tid()
1421 	intel_pt_log("switch: cpu %d tid %d\n", ptq->cpu, tid);  in intel_pt_next_tid()
1423 	err = machine__set_current_tid(pt->machine, ptq->cpu, -1, tid);  in intel_pt_next_tid()
1425 	queue = &pt->queues.queue_array[ptq->queue_nr];  in intel_pt_next_tid()
1428 	ptq->next_tid = -1;  in intel_pt_next_tid()
1433 static inline bool intel_pt_is_switch_ip(struct intel_pt_queue *ptq, u64 ip)  in intel_pt_is_switch_ip()  argument
1435 	struct intel_pt *pt = ptq->pt;  in intel_pt_is_switch_ip()
1438 	       (ptq->flags & PERF_IP_FLAG_BRANCH) &&  in intel_pt_is_switch_ip()
1439 	       !(ptq->flags & (PERF_IP_FLAG_CONDITIONAL | PERF_IP_FLAG_ASYNC |  in intel_pt_is_switch_ip()
1447 static int intel_pt_sample(struct intel_pt_queue *ptq)  in intel_pt_sample()  argument
1449 	const struct intel_pt_state *state = ptq->state;  in intel_pt_sample()
1450 	struct intel_pt *pt = ptq->pt;  in intel_pt_sample()
1453 	if (!ptq->have_sample)  in intel_pt_sample()
1456 	ptq->have_sample = false;  in intel_pt_sample()
1460 			err = intel_pt_synth_cbr_sample(ptq);  in intel_pt_sample()
1465 			err = intel_pt_synth_mwait_sample(ptq);  in intel_pt_sample()
1470 			err = intel_pt_synth_pwre_sample(ptq);  in intel_pt_sample()
1475 			err = intel_pt_synth_exstop_sample(ptq);  in intel_pt_sample()
1480 			err = intel_pt_synth_pwrx_sample(ptq);  in intel_pt_sample()
1487 		err = intel_pt_synth_instruction_sample(ptq);  in intel_pt_sample()
1493 		err = intel_pt_synth_transaction_sample(ptq);  in intel_pt_sample()
1499 		err = intel_pt_synth_ptwrite_sample(ptq);  in intel_pt_sample()
1508 		thread_stack__event(ptq->thread, ptq->flags, state->from_ip,  in intel_pt_sample()
1509 				    state->to_ip, ptq->insn_len,  in intel_pt_sample()
1512 		thread_stack__set_trace_nr(ptq->thread, state->trace_nr);  in intel_pt_sample()
1515 		err = intel_pt_synth_branch_sample(ptq);  in intel_pt_sample()
1521 		intel_pt_update_last_branch_rb(ptq);  in intel_pt_sample()
1523 	if (!ptq->sync_switch)  in intel_pt_sample()
1526 	if (intel_pt_is_switch_ip(ptq, state->to_ip)) {  in intel_pt_sample()
1527 		switch (ptq->switch_state) {  in intel_pt_sample()
1531 			err = intel_pt_next_tid(pt, ptq);  in intel_pt_sample()
1534 			ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
1537 			ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_EVENT;  in intel_pt_sample()
1541 		ptq->switch_state = INTEL_PT_SS_NOT_TRACING;  in intel_pt_sample()
1542 	} else if (ptq->switch_state == INTEL_PT_SS_NOT_TRACING) {  in intel_pt_sample()
1543 		ptq->switch_state = INTEL_PT_SS_UNKNOWN;  in intel_pt_sample()
1544 	} else if (ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_sample()
1546 		   (ptq->flags & PERF_IP_FLAG_CALL)) {  in intel_pt_sample()
1547 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sample()
1613 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_enable_sync_switch()  local
1615 		if (ptq)  in intel_pt_enable_sync_switch()
1616 			ptq->sync_switch = true;  in intel_pt_enable_sync_switch()
1620 static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp)  in intel_pt_run_decoder()  argument
1622 	const struct intel_pt_state *state = ptq->state;  in intel_pt_run_decoder()
1623 	struct intel_pt *pt = ptq->pt;  in intel_pt_run_decoder()
1642 		     ptq->queue_nr, ptq->cpu, ptq->pid, ptq->tid);  in intel_pt_run_decoder()
1644 		err = intel_pt_sample(ptq);  in intel_pt_run_decoder()
1648 		state = intel_pt_decode(ptq->decoder);  in intel_pt_run_decoder()
1652 			if (ptq->sync_switch &&  in intel_pt_run_decoder()
1654 				ptq->sync_switch = false;  in intel_pt_run_decoder()
1655 				intel_pt_next_tid(pt, ptq);  in intel_pt_run_decoder()
1659 							   ptq->cpu, ptq->pid,  in intel_pt_run_decoder()
1660 							   ptq->tid,  in intel_pt_run_decoder()
1668 		ptq->state = state;  in intel_pt_run_decoder()
1669 		ptq->have_sample = true;  in intel_pt_run_decoder()
1670 		intel_pt_sample_flags(ptq);  in intel_pt_run_decoder()
1678 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
1680 		} else if (ptq->sync_switch &&  in intel_pt_run_decoder()
1681 			   ptq->switch_state == INTEL_PT_SS_UNKNOWN &&  in intel_pt_run_decoder()
1682 			   intel_pt_is_switch_ip(ptq, state->to_ip) &&  in intel_pt_run_decoder()
1683 			   ptq->next_tid == -1) {  in intel_pt_run_decoder()
1686 			ptq->timestamp = state->est_timestamp;  in intel_pt_run_decoder()
1687 		} else if (state->timestamp > ptq->timestamp) {  in intel_pt_run_decoder()
1688 			ptq->timestamp = state->timestamp;  in intel_pt_run_decoder()
1691 		if (!pt->timeless_decoding && ptq->timestamp >= *timestamp) {  in intel_pt_run_decoder()
1692 			*timestamp = ptq->timestamp;  in intel_pt_run_decoder()
1716 		struct intel_pt_queue *ptq;  in intel_pt_process_queues()  local
1726 		ptq = queue->priv;  in intel_pt_process_queues()
1744 		ret = intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_queues()
1756 			ptq->on_heap = false;  in intel_pt_process_queues()
1772 		struct intel_pt_queue *ptq = queue->priv;  in intel_pt_process_timeless_queues()  local
1774 		if (ptq && (tid == -1 || ptq->tid == tid)) {  in intel_pt_process_timeless_queues()
1775 			ptq->time = time_;  in intel_pt_process_timeless_queues()
1777 			intel_pt_run_decoder(ptq, &ts);  in intel_pt_process_timeless_queues()
1820 	struct intel_pt_queue *ptq;  in intel_pt_sync_switch()  local
1826 	ptq = intel_pt_cpu_to_ptq(pt, cpu);  in intel_pt_sync_switch()
1827 	if (!ptq || !ptq->sync_switch)  in intel_pt_sync_switch()
1830 	switch (ptq->switch_state) {  in intel_pt_sync_switch()
1832 		ptq->next_tid = -1;  in intel_pt_sync_switch()
1836 		ptq->next_tid = tid;  in intel_pt_sync_switch()
1837 		ptq->switch_state = INTEL_PT_SS_EXPECTING_SWITCH_IP;  in intel_pt_sync_switch()
1840 		if (!ptq->on_heap) {  in intel_pt_sync_switch()
1841 			ptq->timestamp = perf_time_to_tsc(timestamp,  in intel_pt_sync_switch()
1843 			err = auxtrace_heap__add(&pt->heap, ptq->queue_nr,  in intel_pt_sync_switch()
1844 						 ptq->timestamp);  in intel_pt_sync_switch()
1847 			ptq->on_heap = true;  in intel_pt_sync_switch()
1849 		ptq->switch_state = INTEL_PT_SS_TRACING;  in intel_pt_sync_switch()
1852 		ptq->next_tid = tid;  in intel_pt_sync_switch()