111 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
260 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,  in intel_pt_print_once()  argument
265 	if (decoder->print_once & bit)  in intel_pt_print_once()
267 	decoder->print_once |= bit;  in intel_pt_print_once()
279 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)  in intel_pt_setup_period()  argument
281 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {  in intel_pt_setup_period()
284 		period = intel_pt_lower_power_of_2(decoder->period);  in intel_pt_setup_period()
285 		decoder->period_mask  = ~(period - 1);  in intel_pt_setup_period()
286 		decoder->period_ticks = period;  in intel_pt_setup_period()
299 	struct intel_pt_decoder *decoder;  in intel_pt_decoder_new()  local
304 	decoder = zalloc(sizeof(struct intel_pt_decoder));  in intel_pt_decoder_new()
305 	if (!decoder)  in intel_pt_decoder_new()
308 	decoder->get_trace          = params->get_trace;  in intel_pt_decoder_new()
309 	decoder->walk_insn          = params->walk_insn;  in intel_pt_decoder_new()
310 	decoder->pgd_ip             = params->pgd_ip;  in intel_pt_decoder_new()
311 	decoder->lookahead          = params->lookahead;  in intel_pt_decoder_new()
312 	decoder->findnew_vmcs_info  = params->findnew_vmcs_info;  in intel_pt_decoder_new()
313 	decoder->data               = params->data;  in intel_pt_decoder_new()
314 	decoder->return_compression = params->return_compression;  in intel_pt_decoder_new()
315 	decoder->branch_enable      = params->branch_enable;  in intel_pt_decoder_new()
316 	decoder->hop                = params->quick >= 1;  in intel_pt_decoder_new()
317 	decoder->leap               = params->quick >= 2;  in intel_pt_decoder_new()
318 	decoder->vm_time_correlation = params->vm_time_correlation;  in intel_pt_decoder_new()
319 	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;  in intel_pt_decoder_new()
320 	decoder->first_timestamp    = params->first_timestamp;  in intel_pt_decoder_new()
321 	decoder->last_reliable_timestamp = params->first_timestamp;  in intel_pt_decoder_new()
322 	decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;  in intel_pt_decoder_new()
324 	decoder->flags              = params->flags;  in intel_pt_decoder_new()
326 	decoder->ctl                = params->ctl;  in intel_pt_decoder_new()
327 	decoder->period             = params->period;  in intel_pt_decoder_new()
328 	decoder->period_type        = params->period_type;  in intel_pt_decoder_new()
330 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
331 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
333 	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);  in intel_pt_decoder_new()
335 	intel_pt_setup_period(decoder);  in intel_pt_decoder_new()
337 	decoder->mtc_shift = params->mtc_period;  in intel_pt_decoder_new()
338 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;  in intel_pt_decoder_new()
340 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;  in intel_pt_decoder_new()
341 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
343 	if (!decoder->tsc_ctc_ratio_n)  in intel_pt_decoder_new()
344 		decoder->tsc_ctc_ratio_d = 0;  in intel_pt_decoder_new()
346 	if (decoder->tsc_ctc_ratio_d) {  in intel_pt_decoder_new()
347 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))  in intel_pt_decoder_new()
348 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /  in intel_pt_decoder_new()
349 						decoder->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
358 	decoder->tsc_slip = 0x10000;  in intel_pt_decoder_new()
360 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);  in intel_pt_decoder_new()
361 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);  in intel_pt_decoder_new()
362 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
363 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);  in intel_pt_decoder_new()
364 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);  in intel_pt_decoder_new()
366 	if (decoder->hop)  in intel_pt_decoder_new()
369 	return decoder;  in intel_pt_decoder_new()
372 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,  in intel_pt_set_first_timestamp()  argument
375 	decoder->first_timestamp = first_timestamp;  in intel_pt_set_first_timestamp()
449 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)  in intel_pt_decoder_free()  argument
451 	intel_pt_free_stack(&decoder->stack);  in intel_pt_decoder_free()
452 	free(decoder);  in intel_pt_decoder_free()
536 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_last_ip()  argument
538 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);  in intel_pt_set_last_ip()
539 	decoder->have_last_ip = true;  in intel_pt_set_last_ip()
542 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_ip()  argument
544 	intel_pt_set_last_ip(decoder);  in intel_pt_set_ip()
545 	decoder->ip = decoder->last_ip;  in intel_pt_set_ip()
548 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)  in intel_pt_decoder_log_packet()  argument
550 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,  in intel_pt_decoder_log_packet()
551 			    decoder->buf);  in intel_pt_decoder_log_packet()
554 static int intel_pt_bug(struct intel_pt_decoder *decoder)  in intel_pt_bug()  argument
557 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_bug()
561 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)  in intel_pt_clear_tx_flags()  argument
563 	decoder->tx_flags = 0;  in intel_pt_clear_tx_flags()
566 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)  in intel_pt_update_in_tx()  argument
568 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;  in intel_pt_update_in_tx()
571 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)  in intel_pt_update_pip()  argument
573 	decoder->pip_payload = decoder->packet.payload;  in intel_pt_update_pip()
576 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)  in intel_pt_update_nr()  argument
578 	decoder->next_nr = decoder->pip_payload & 1;  in intel_pt_update_nr()
581 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)  in intel_pt_set_nr()  argument
583 	decoder->nr = decoder->pip_payload & 1;  in intel_pt_set_nr()
584 	decoder->next_nr = decoder->nr;  in intel_pt_set_nr()
587 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)  in intel_pt_set_pip()  argument
589 	intel_pt_update_pip(decoder);  in intel_pt_set_pip()
590 	intel_pt_set_nr(decoder);  in intel_pt_set_pip()
593 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)  in intel_pt_bad_packet()  argument
595 	intel_pt_clear_tx_flags(decoder);  in intel_pt_bad_packet()
596 	decoder->have_tma = false;  in intel_pt_bad_packet()
597 	decoder->pkt_len = 1;  in intel_pt_bad_packet()
598 	decoder->pkt_step = 1;  in intel_pt_bad_packet()
599 	intel_pt_decoder_log_packet(decoder);  in intel_pt_bad_packet()
600 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {  in intel_pt_bad_packet()
602 		decoder->pkt_state = INTEL_PT_STATE_ERR1;  in intel_pt_bad_packet()
607 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)  in intel_pt_update_sample_time()  argument
609 	decoder->sample_timestamp = decoder->timestamp;  in intel_pt_update_sample_time()
610 	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_update_sample_time()
613 static void intel_pt_reposition(struct intel_pt_decoder *decoder)  in intel_pt_reposition()  argument
615 	decoder->ip = 0;  in intel_pt_reposition()
616 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_reposition()
617 	decoder->timestamp = 0;  in intel_pt_reposition()
618 	decoder->have_tma = false;  in intel_pt_reposition()
621 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)  in intel_pt_get_data()  argument
626 	decoder->pkt_step = 0;  in intel_pt_get_data()
629 	ret = decoder->get_trace(&buffer, decoder->data);  in intel_pt_get_data()
632 	decoder->buf = buffer.buf;  in intel_pt_get_data()
633 	decoder->len = buffer.len;  in intel_pt_get_data()
634 	if (!decoder->len) {  in intel_pt_get_data()
638 	decoder->buf_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
640 		intel_pt_reposition(decoder);  in intel_pt_get_data()
641 		decoder->ref_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
642 		decoder->state.trace_nr = buffer.trace_nr;  in intel_pt_get_data()
643 		decoder->vm_tm_corr_same_buf = false;  in intel_pt_get_data()
645 			     decoder->ref_timestamp);  in intel_pt_get_data()
652 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,  in intel_pt_get_next_data()  argument
655 	if (!decoder->next_buf)  in intel_pt_get_next_data()
656 		return intel_pt_get_data(decoder, reposition);  in intel_pt_get_next_data()
658 	decoder->buf = decoder->next_buf;  in intel_pt_get_next_data()
659 	decoder->len = decoder->next_len;  in intel_pt_get_next_data()
660 	decoder->next_buf = 0;  in intel_pt_get_next_data()
661 	decoder->next_len = 0;  in intel_pt_get_next_data()
665 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_split_packet()  argument
667 	unsigned char *buf = decoder->temp_buf;  in intel_pt_get_split_packet()
671 	old_len = decoder->len;  in intel_pt_get_split_packet()
672 	len = decoder->len;  in intel_pt_get_split_packet()
673 	memcpy(buf, decoder->buf, len);  in intel_pt_get_split_packet()
675 	ret = intel_pt_get_data(decoder, false);  in intel_pt_get_split_packet()
677 		decoder->pos += old_len;  in intel_pt_get_split_packet()
682 	if (n > decoder->len)  in intel_pt_get_split_packet()
683 		n = decoder->len;  in intel_pt_get_split_packet()
684 	memcpy(buf + len, decoder->buf, n);  in intel_pt_get_split_packet()
687 	decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_split_packet()
688 	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_split_packet()
690 		decoder->next_buf = decoder->buf;  in intel_pt_get_split_packet()
691 		decoder->next_len = decoder->len;  in intel_pt_get_split_packet()
692 		decoder->buf = buf;  in intel_pt_get_split_packet()
693 		decoder->len = old_len;  in intel_pt_get_split_packet()
694 		return intel_pt_bad_packet(decoder);  in intel_pt_get_split_packet()
697 	decoder->next_buf = decoder->buf + (ret - old_len);  in intel_pt_get_split_packet()
698 	decoder->next_len = decoder->len - (ret - old_len);  in intel_pt_get_split_packet()
700 	decoder->buf = buf;  in intel_pt_get_split_packet()
701 	decoder->len = ret;  in intel_pt_get_split_packet()
707 	struct intel_pt_decoder	  *decoder;  member
718 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,  in intel_pt_pkt_lookahead()  argument
722 	const unsigned char *buf = decoder->buf;  in intel_pt_pkt_lookahead()
723 	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;  in intel_pt_pkt_lookahead()
724 	size_t len = decoder->len;  in intel_pt_pkt_lookahead()
727 	pkt_info.decoder          = decoder;  in intel_pt_pkt_lookahead()
728 	pkt_info.pos              = decoder->pos;  in intel_pt_pkt_lookahead()
729 	pkt_info.pkt_len          = decoder->pkt_step;  in intel_pt_pkt_lookahead()
730 	pkt_info.last_packet_type = decoder->last_packet_type;  in intel_pt_pkt_lookahead()
796 	struct intel_pt_decoder *decoder = pkt_info->decoder;  in intel_pt_calc_cyc_cb()  local
829 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {  in intel_pt_calc_cyc_cb()
831 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_cyc_cb()
838 		data->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_cyc_cb()
841 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
843 				data->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
847 					decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
848 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
871 		    data->timestamp - timestamp < decoder->tsc_slip)  in intel_pt_calc_cyc_cb()
888 		if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_cyc_cb()
893 		ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_cyc_cb()
895 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_cyc_cb()
898 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
899 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
902 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
903 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
921 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cyc_cb()
937 	if (!data->cbr && decoder->cbr) {  in intel_pt_calc_cyc_cb()
938 		data->cbr = decoder->cbr;  in intel_pt_calc_cyc_cb()
939 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_cb()
945 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;  in intel_pt_calc_cyc_cb()
954 	decoder->calc_cyc_to_tsc = cyc_to_tsc;  in intel_pt_calc_cyc_cb()
955 	decoder->have_calc_cyc_to_tsc = true;  in intel_pt_calc_cyc_cb()
968 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_cyc_to_tsc()  argument
974 		.last_mtc       = decoder->last_mtc,  in intel_pt_calc_cyc_to_tsc()
975 		.ctc_timestamp  = decoder->ctc_timestamp,  in intel_pt_calc_cyc_to_tsc()
976 		.ctc_delta      = decoder->ctc_delta,  in intel_pt_calc_cyc_to_tsc()
977 		.tsc_timestamp  = decoder->tsc_timestamp,  in intel_pt_calc_cyc_to_tsc()
978 		.timestamp      = decoder->timestamp,  in intel_pt_calc_cyc_to_tsc()
979 		.have_tma       = decoder->have_tma,  in intel_pt_calc_cyc_to_tsc()
980 		.fixup_last_mtc = decoder->fixup_last_mtc,  in intel_pt_calc_cyc_to_tsc()
993 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);  in intel_pt_calc_cyc_to_tsc()
996 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_next_packet()  argument
1000 	decoder->last_packet_type = decoder->packet.type;  in intel_pt_get_next_packet()
1003 		decoder->pos += decoder->pkt_step;  in intel_pt_get_next_packet()
1004 		decoder->buf += decoder->pkt_step;  in intel_pt_get_next_packet()
1005 		decoder->len -= decoder->pkt_step;  in intel_pt_get_next_packet()
1007 		if (!decoder->len) {  in intel_pt_get_next_packet()
1008 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_next_packet()
1013 		decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_next_packet()
1014 		ret = intel_pt_get_packet(decoder->buf, decoder->len,  in intel_pt_get_next_packet()
1015 					  &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_next_packet()
1017 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {  in intel_pt_get_next_packet()
1018 			ret = intel_pt_get_split_packet(decoder);  in intel_pt_get_next_packet()
1023 			return intel_pt_bad_packet(decoder);  in intel_pt_get_next_packet()
1025 		decoder->pkt_len = ret;  in intel_pt_get_next_packet()
1026 		decoder->pkt_step = ret;  in intel_pt_get_next_packet()
1027 		intel_pt_decoder_log_packet(decoder);  in intel_pt_get_next_packet()
1028 	} while (decoder->packet.type == INTEL_PT_PAD);  in intel_pt_get_next_packet()
1033 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)  in intel_pt_next_period()  argument
1037 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_next_period()
1038 	masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1039 	if (decoder->continuous_period) {  in intel_pt_next_period()
1040 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_next_period()
1044 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1045 		if (masked_timestamp > decoder->last_masked_timestamp) {  in intel_pt_next_period()
1046 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_next_period()
1047 			decoder->continuous_period = true;  in intel_pt_next_period()
1051 	if (masked_timestamp < decoder->last_masked_timestamp)  in intel_pt_next_period()
1052 		return decoder->period_ticks;  in intel_pt_next_period()
1054 	return decoder->period_ticks - (timestamp - masked_timestamp);  in intel_pt_next_period()
1057 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)  in intel_pt_next_sample()  argument
1059 	switch (decoder->period_type) {  in intel_pt_next_sample()
1061 		return decoder->period - decoder->period_insn_cnt;  in intel_pt_next_sample()
1063 		return intel_pt_next_period(decoder);  in intel_pt_next_sample()
1071 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_insn()  argument
1075 	switch (decoder->period_type) {  in intel_pt_sample_insn()
1077 		decoder->period_insn_cnt = 0;  in intel_pt_sample_insn()
1080 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_sample_insn()
1081 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_sample_insn()
1082 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_sample_insn()
1083 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_sample_insn()
1085 			decoder->last_masked_timestamp += decoder->period_ticks;  in intel_pt_sample_insn()
1093 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_insn()
1096 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,  in intel_pt_walk_insn()  argument
1102 	if (!decoder->mtc_insn)  in intel_pt_walk_insn()
1103 		decoder->mtc_insn = true;  in intel_pt_walk_insn()
1105 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_walk_insn()
1107 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,  in intel_pt_walk_insn()
1108 				 max_insn_cnt, decoder->data);  in intel_pt_walk_insn()
1110 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1111 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1112 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1113 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1116 		decoder->no_progress = 0;  in intel_pt_walk_insn()
1117 		decoder->pkt_state = INTEL_PT_STATE_ERR2;  in intel_pt_walk_insn()
1119 				decoder->ip);  in intel_pt_walk_insn()
1125 	if (ip && decoder->ip == ip) {  in intel_pt_walk_insn()
1131 		intel_pt_sample_insn(decoder);  in intel_pt_walk_insn()
1134 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_insn()
1135 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1136 		decoder->state.to_ip = 0;  in intel_pt_walk_insn()
1137 		decoder->ip += intel_pt_insn->length;  in intel_pt_walk_insn()
1146 			err = intel_pt_push(&decoder->stack, decoder->ip +  in intel_pt_walk_insn()
1152 		decoder->ret_addr = intel_pt_pop(&decoder->stack);  in intel_pt_walk_insn()
1156 		int cnt = decoder->no_progress++;  in intel_pt_walk_insn()
1158 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1159 		decoder->ip += intel_pt_insn->length +  in intel_pt_walk_insn()
1161 		decoder->state.to_ip = decoder->ip;  in intel_pt_walk_insn()
1173 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1174 				decoder->stuck_ip_prd = 1;  in intel_pt_walk_insn()
1175 				decoder->stuck_ip_cnt = 1;  in intel_pt_walk_insn()
1176 			} else if (cnt > decoder->max_loops ||  in intel_pt_walk_insn()
1177 				   decoder->state.to_ip == decoder->stuck_ip) {  in intel_pt_walk_insn()
1179 						decoder->state.to_ip);  in intel_pt_walk_insn()
1180 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_insn()
1183 			} else if (!--decoder->stuck_ip_cnt) {  in intel_pt_walk_insn()
1184 				decoder->stuck_ip_prd += 1;  in intel_pt_walk_insn()
1185 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;  in intel_pt_walk_insn()
1186 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1192 	decoder->no_progress = 0;  in intel_pt_walk_insn()
1194 	decoder->state.insn_op = intel_pt_insn->op;  in intel_pt_walk_insn()
1195 	decoder->state.insn_len = intel_pt_insn->length;  in intel_pt_walk_insn()
1196 	memcpy(decoder->state.insn, intel_pt_insn->buf,  in intel_pt_walk_insn()
1199 	if (decoder->tx_flags & INTEL_PT_IN_TX)  in intel_pt_walk_insn()
1200 		decoder->state.flags |= INTEL_PT_IN_TX;  in intel_pt_walk_insn()
1205 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)  in intel_pt_fup_event()  argument
1209 	if (decoder->set_fup_tx_flags) {  in intel_pt_fup_event()
1210 		decoder->set_fup_tx_flags = false;  in intel_pt_fup_event()
1211 		decoder->tx_flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1212 		decoder->state.type = INTEL_PT_TRANSACTION;  in intel_pt_fup_event()
1213 		if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)  in intel_pt_fup_event()
1214 			decoder->state.type |= INTEL_PT_BRANCH;  in intel_pt_fup_event()
1215 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1216 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1217 		decoder->state.flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1220 	if (decoder->set_fup_ptw) {  in intel_pt_fup_event()
1221 		decoder->set_fup_ptw = false;  in intel_pt_fup_event()
1222 		decoder->state.type = INTEL_PT_PTW;  in intel_pt_fup_event()
1223 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1224 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1225 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1226 		decoder->state.ptw_payload = decoder->fup_ptw_payload;  in intel_pt_fup_event()
1229 	if (decoder->set_fup_mwait) {  in intel_pt_fup_event()
1230 		decoder->set_fup_mwait = false;  in intel_pt_fup_event()
1231 		decoder->state.type = INTEL_PT_MWAIT_OP;  in intel_pt_fup_event()
1232 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1233 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1234 		decoder->state.mwait_payload = decoder->fup_mwait_payload;  in intel_pt_fup_event()
1237 	if (decoder->set_fup_pwre) {  in intel_pt_fup_event()
1238 		decoder->set_fup_pwre = false;  in intel_pt_fup_event()
1239 		decoder->state.type |= INTEL_PT_PWR_ENTRY;  in intel_pt_fup_event()
1240 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1241 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1242 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1243 		decoder->state.pwre_payload = decoder->fup_pwre_payload;  in intel_pt_fup_event()
1246 	if (decoder->set_fup_exstop) {  in intel_pt_fup_event()
1247 		decoder->set_fup_exstop = false;  in intel_pt_fup_event()
1248 		decoder->state.type |= INTEL_PT_EX_STOP;  in intel_pt_fup_event()
1249 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1250 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1251 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1252 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1255 	if (decoder->set_fup_bep) {  in intel_pt_fup_event()
1256 		decoder->set_fup_bep = false;  in intel_pt_fup_event()
1257 		decoder->state.type |= INTEL_PT_BLK_ITEMS;  in intel_pt_fup_event()
1258 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1259 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1260 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1266 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,  in intel_pt_fup_with_nlip()  argument
1270 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&  in intel_pt_fup_with_nlip()
1272 	       ip == decoder->ip + intel_pt_insn->length;  in intel_pt_fup_with_nlip()
1275 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup()  argument
1281 	ip = decoder->last_ip;  in intel_pt_walk_fup()
1284 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);  in intel_pt_walk_fup()
1288 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {  in intel_pt_walk_fup()
1289 			bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;  in intel_pt_walk_fup()
1291 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_fup()
1292 			if (intel_pt_fup_event(decoder) && no_tip)  in intel_pt_walk_fup()
1296 		decoder->set_fup_tx_flags = false;  in intel_pt_walk_fup()
1302 					decoder->ip);  in intel_pt_walk_fup()
1303 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1309 					decoder->ip);  in intel_pt_walk_fup()
1310 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1314 		intel_pt_bug(decoder);  in intel_pt_walk_fup()
1318 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_tip()  argument
1323 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tip()
1325 	    decoder->pgd_ip &&  in intel_pt_walk_tip()
1326 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1327 	    (decoder->state.type & INTEL_PT_BRANCH) &&  in intel_pt_walk_tip()
1328 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {  in intel_pt_walk_tip()
1330 		decoder->no_progress = 0;  in intel_pt_walk_tip()
1331 		decoder->pge = false;  in intel_pt_walk_tip()
1332 		decoder->continuous_period = false;  in intel_pt_walk_tip()
1333 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1334 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1335 		intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1343 	intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1346 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {  in intel_pt_walk_tip()
1347 			decoder->pge = false;  in intel_pt_walk_tip()
1348 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1349 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1350 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1351 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1352 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1354 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1355 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1357 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1359 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1360 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1361 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1362 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1364 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1365 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1372 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +  in intel_pt_walk_tip()
1375 		if (decoder->pgd_ip &&  in intel_pt_walk_tip()
1376 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1377 		    decoder->pgd_ip(to_ip, decoder->data)) {  in intel_pt_walk_tip()
1379 			decoder->pge = false;  in intel_pt_walk_tip()
1380 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1381 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1382 			decoder->ip = to_ip;  in intel_pt_walk_tip()
1383 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1384 			decoder->state.to_ip = to_ip;  in intel_pt_walk_tip()
1385 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1389 				decoder->ip);  in intel_pt_walk_tip()
1390 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_tip()
1394 	return intel_pt_bug(decoder);  in intel_pt_walk_tip()
1397 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)  in intel_pt_walk_tnt()  argument
1403 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tnt()
1410 			if (!decoder->return_compression) {  in intel_pt_walk_tnt()
1412 						decoder->ip);  in intel_pt_walk_tnt()
1413 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1416 			if (!decoder->ret_addr) {  in intel_pt_walk_tnt()
1418 						decoder->ip);  in intel_pt_walk_tnt()
1419 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1422 			if (!(decoder->tnt.payload & BIT63)) {  in intel_pt_walk_tnt()
1424 						decoder->ip);  in intel_pt_walk_tnt()
1425 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1428 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1429 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1430 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1432 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1433 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1434 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1435 			decoder->ip = decoder->ret_addr;  in intel_pt_walk_tnt()
1436 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1442 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_tnt()
1445 			if (decoder->packet.type != INTEL_PT_TIP ||  in intel_pt_walk_tnt()
1446 			    decoder->packet.count == 0) {  in intel_pt_walk_tnt()
1448 						decoder->ip);  in intel_pt_walk_tnt()
1449 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1450 				decoder->pkt_step = 0;  in intel_pt_walk_tnt()
1453 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_tnt()
1454 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1455 			decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tnt()
1456 			decoder->ip = decoder->last_ip;  in intel_pt_walk_tnt()
1457 			intel_pt_update_nr(decoder);  in intel_pt_walk_tnt()
1462 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1463 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1464 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1466 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1467 			if (decoder->tnt.payload & BIT63) {  in intel_pt_walk_tnt()
1468 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1469 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1470 				decoder->ip += intel_pt_insn.length +  in intel_pt_walk_tnt()
1472 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1476 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {  in intel_pt_walk_tnt()
1477 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1478 				decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_tnt()
1479 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1480 				decoder->state.to_ip = 0;  in intel_pt_walk_tnt()
1481 				decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1484 			decoder->sample_cyc = false;  in intel_pt_walk_tnt()
1485 			decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1486 			if (!decoder->tnt.count) {  in intel_pt_walk_tnt()
1487 				intel_pt_update_sample_time(decoder);  in intel_pt_walk_tnt()
1490 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1494 		return intel_pt_bug(decoder);  in intel_pt_walk_tnt()
1498 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)  in intel_pt_mode_tsx()  argument
1503 	fup_tx_flags = decoder->packet.payload &  in intel_pt_mode_tsx()
1505 	err = intel_pt_get_next_packet(decoder);  in intel_pt_mode_tsx()
1508 	if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_mode_tsx()
1509 		decoder->fup_tx_flags = fup_tx_flags;  in intel_pt_mode_tsx()
1510 		decoder->set_fup_tx_flags = true;  in intel_pt_mode_tsx()
1511 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))  in intel_pt_mode_tsx()
1515 				decoder->pos);  in intel_pt_mode_tsx()
1516 		intel_pt_update_in_tx(decoder);  in intel_pt_mode_tsx()
1537 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,  in intel_pt_time_in_range()  argument
1540 	uint64_t max_timestamp = decoder->buf_timestamp;  in intel_pt_time_in_range()
1543 		max_timestamp = decoder->last_reliable_timestamp +  in intel_pt_time_in_range()
1546 	return timestamp >= decoder->last_reliable_timestamp &&  in intel_pt_time_in_range()
1547 	       timestamp < decoder->buf_timestamp;  in intel_pt_time_in_range()
1550 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_tsc_timestamp()  argument
1555 	decoder->have_tma = false;  in intel_pt_calc_tsc_timestamp()
1557 	if (decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1558 		timestamp = intel_pt_8b_tsc(decoder->packet.payload,  in intel_pt_calc_tsc_timestamp()
1559 					    decoder->ref_timestamp);  in intel_pt_calc_tsc_timestamp()
1560 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1561 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1562 		decoder->ref_timestamp = 0;  in intel_pt_calc_tsc_timestamp()
1563 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1564 	} else if (decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1565 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1566 			    (decoder->timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1567 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1568 		if (timestamp < decoder->timestamp &&  in intel_pt_calc_tsc_timestamp()
1569 		    decoder->timestamp - timestamp < decoder->tsc_slip) {  in intel_pt_calc_tsc_timestamp()
1572 			timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1574 		if (timestamp < decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1575 			if (!decoder->buf_timestamp ||  in intel_pt_calc_tsc_timestamp()
1576 			    (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {  in intel_pt_calc_tsc_timestamp()
1579 				decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1582 				timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1586 		if (decoder->vm_time_correlation &&  in intel_pt_calc_tsc_timestamp()
1587 		    (bad || !intel_pt_time_in_range(decoder, timestamp)) &&  in intel_pt_calc_tsc_timestamp()
1588 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_calc_tsc_timestamp()
1590 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1591 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1594 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_tsc_timestamp()
1595 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1596 		decoder->cycle_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1597 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_tsc_timestamp()
1598 		intel_pt_calc_cyc_to_tsc(decoder, false);  in intel_pt_calc_tsc_timestamp()
1601 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_tsc_timestamp()
1604 static int intel_pt_overflow(struct intel_pt_decoder *decoder)  in intel_pt_overflow()  argument
1607 	intel_pt_clear_tx_flags(decoder);  in intel_pt_overflow()
1608 	intel_pt_set_nr(decoder);  in intel_pt_overflow()
1609 	decoder->timestamp_insn_cnt = 0;  in intel_pt_overflow()
1610 	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_overflow()
1611 	decoder->overflow = true;  in intel_pt_overflow()
1615 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_pge()  argument
1617 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_pge()
1620 	decoder->cyc_cnt_timestamp = decoder->timestamp;  in intel_pt_mtc_cyc_cnt_pge()
1621 	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_pge()
1624 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_cbr()  argument
1626 	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;  in intel_pt_mtc_cyc_cnt_cbr()
1628 	if (decoder->pge)  in intel_pt_mtc_cyc_cnt_cbr()
1629 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_mtc_cyc_cnt_cbr()
1632 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_upd()  argument
1636 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_upd()
1639 	decoder->sample_cyc = true;  in intel_pt_mtc_cyc_cnt_upd()
1641 	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)  in intel_pt_mtc_cyc_cnt_upd()
1644 	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;  in intel_pt_mtc_cyc_cnt_upd()
1645 	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1647 	if (tot_cyc_cnt > decoder->tot_cyc_cnt)  in intel_pt_mtc_cyc_cnt_upd()
1648 		decoder->tot_cyc_cnt = tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1651 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)  in intel_pt_calc_tma()  argument
1653 	uint32_t ctc = decoder->packet.payload;  in intel_pt_calc_tma()
1654 	uint32_t fc = decoder->packet.count;  in intel_pt_calc_tma()
1655 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_tma()
1657 	if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_tma()
1660 	if (decoder->pge && !decoder->in_psb)  in intel_pt_calc_tma()
1661 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_calc_tma()
1663 		intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_tma()
1665 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_tma()
1666 	decoder->last_ctc = ctc - ctc_rem;  in intel_pt_calc_tma()
1667 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;  in intel_pt_calc_tma()
1668 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_tma()
1669 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_tma()
1671 		decoder->ctc_timestamp -= multdiv(ctc_rem,  in intel_pt_calc_tma()
1672 						  decoder->tsc_ctc_ratio_n,  in intel_pt_calc_tma()
1673 						  decoder->tsc_ctc_ratio_d);  in intel_pt_calc_tma()
1675 	decoder->ctc_delta = 0;  in intel_pt_calc_tma()
1676 	decoder->have_tma = true;  in intel_pt_calc_tma()
1677 	decoder->fixup_last_mtc = true;  in intel_pt_calc_tma()
1679 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);  in intel_pt_calc_tma()
1682 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_mtc_timestamp()  argument
1687 	if (!decoder->have_tma)  in intel_pt_calc_mtc_timestamp()
1690 	mtc = decoder->packet.payload;  in intel_pt_calc_mtc_timestamp()
1692 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {  in intel_pt_calc_mtc_timestamp()
1693 		decoder->fixup_last_mtc = false;  in intel_pt_calc_mtc_timestamp()
1694 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_mtc_timestamp()
1695 					&decoder->last_mtc);  in intel_pt_calc_mtc_timestamp()
1698 	if (mtc > decoder->last_mtc)  in intel_pt_calc_mtc_timestamp()
1699 		mtc_delta = mtc - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1701 		mtc_delta = mtc + 256 - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1703 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_mtc_timestamp()
1705 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_mtc_timestamp()
1706 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1707 			    decoder->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_mtc_timestamp()
1709 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1710 			    multdiv(decoder->ctc_delta,  in intel_pt_calc_mtc_timestamp()
1711 				    decoder->tsc_ctc_ratio_n,  in intel_pt_calc_mtc_timestamp()
1712 				    decoder->tsc_ctc_ratio_d);  in intel_pt_calc_mtc_timestamp()
1715 	if (timestamp < decoder->timestamp)  in intel_pt_calc_mtc_timestamp()
1717 			     timestamp, decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1719 		decoder->timestamp = timestamp;  in intel_pt_calc_mtc_timestamp()
1721 	intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_mtc_timestamp()
1723 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1724 	decoder->last_mtc = mtc;  in intel_pt_calc_mtc_timestamp()
1726 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_mtc_timestamp()
1727 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_mtc_timestamp()
1728 		decoder->cycle_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1729 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_mtc_timestamp()
1730 		intel_pt_calc_cyc_to_tsc(decoder, true);  in intel_pt_calc_mtc_timestamp()
1733 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1736 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)  in intel_pt_calc_cbr()  argument
1738 	unsigned int cbr = decoder->packet.payload & 0xff;  in intel_pt_calc_cbr()
1740 	decoder->cbr_payload = decoder->packet.payload;  in intel_pt_calc_cbr()
1742 	if (decoder->cbr == cbr)  in intel_pt_calc_cbr()
1745 	decoder->cbr = cbr;  in intel_pt_calc_cbr()
1746 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cbr()
1748 	intel_pt_mtc_cyc_cnt_cbr(decoder);  in intel_pt_calc_cbr()
1751 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_cyc_timestamp()  argument
1753 	uint64_t timestamp = decoder->cyc_ref_timestamp;  in intel_pt_calc_cyc_timestamp()
1755 	decoder->have_cyc = true;  in intel_pt_calc_cyc_timestamp()
1757 	decoder->cycle_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
1758 	if (decoder->pge)  in intel_pt_calc_cyc_timestamp()
1759 		decoder->tot_cyc_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
1760 	decoder->sample_cyc = true;  in intel_pt_calc_cyc_timestamp()
1762 	if (!decoder->cyc_ref_timestamp)  in intel_pt_calc_cyc_timestamp()
1765 	if (decoder->have_calc_cyc_to_tsc)  in intel_pt_calc_cyc_timestamp()
1766 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1767 	else if (decoder->cbr)  in intel_pt_calc_cyc_timestamp()
1768 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1772 	if (timestamp < decoder->timestamp)  in intel_pt_calc_cyc_timestamp()
1774 			     timestamp, decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
1776 		decoder->timestamp = timestamp;  in intel_pt_calc_cyc_timestamp()
1778 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_cyc_timestamp()
1780 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
1783 static void intel_pt_bbp(struct intel_pt_decoder *decoder)  in intel_pt_bbp()  argument
1785 	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {  in intel_pt_bbp()
1786 		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));  in intel_pt_bbp()
1787 		decoder->state.items.is_32_bit = false;  in intel_pt_bbp()
1789 	decoder->blk_type = decoder->packet.payload;  in intel_pt_bbp()
1790 	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);  in intel_pt_bbp()
1791 	if (decoder->blk_type == INTEL_PT_GP_REGS)  in intel_pt_bbp()
1792 		decoder->state.items.is_32_bit = decoder->packet.count;  in intel_pt_bbp()
1793 	if (decoder->blk_type_pos < 0) {  in intel_pt_bbp()
1795 			     decoder->blk_type);  in intel_pt_bbp()
1796 	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {  in intel_pt_bbp()
1798 			     decoder->blk_type);  in intel_pt_bbp()
1802 static void intel_pt_bip(struct intel_pt_decoder *decoder)  in intel_pt_bip()  argument
1804 	uint32_t id = decoder->packet.count;  in intel_pt_bip()
1806 	int pos = decoder->blk_type_pos;  in intel_pt_bip()
1810 			     id, decoder->blk_type);  in intel_pt_bip()
1814 	if (decoder->state.items.mask[pos] & bit) {  in intel_pt_bip()
1816 			     id, decoder->blk_type);  in intel_pt_bip()
1819 	decoder->state.items.mask[pos] |= bit;  in intel_pt_bip()
1820 	decoder->state.items.val[pos][id] = decoder->packet.payload;  in intel_pt_bip()
1824 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)  in intel_pt_walk_psbend()  argument
1828 	decoder->in_psb = true;  in intel_pt_walk_psbend()
1831 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psbend()
1835 		switch (decoder->packet.type) {  in intel_pt_walk_psbend()
1858 			decoder->have_tma = false;  in intel_pt_walk_psbend()
1864 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psbend()
1868 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psbend()
1872 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psbend()
1876 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psbend()
1880 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psbend()
1884 			intel_pt_set_pip(decoder);  in intel_pt_walk_psbend()
1888 			decoder->pge = true;  in intel_pt_walk_psbend()
1889 			if (decoder->packet.count) {  in intel_pt_walk_psbend()
1890 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_psbend()
1891 				decoder->psb_ip = decoder->last_ip;  in intel_pt_walk_psbend()
1896 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psbend()
1900 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psbend()
1901 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_psbend()
1902 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_psbend()
1906 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psbend()
1917 	decoder->in_psb = false;  in intel_pt_walk_psbend()
1922 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup_tip()  argument
1926 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {  in intel_pt_walk_fup_tip()
1927 		decoder->tx_flags = 0;  in intel_pt_walk_fup_tip()
1928 		decoder->state.flags &= ~INTEL_PT_IN_TX;  in intel_pt_walk_fup_tip()
1929 		decoder->state.flags |= INTEL_PT_ABORT_TX;  in intel_pt_walk_fup_tip()
1931 		decoder->state.flags |= INTEL_PT_ASYNC;  in intel_pt_walk_fup_tip()
1935 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_fup_tip()
1939 		switch (decoder->packet.type) {  in intel_pt_walk_fup_tip()
1961 			decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_fup_tip()
1962 			decoder->pkt_step = 0;  in intel_pt_walk_fup_tip()
1966 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_fup_tip()
1970 			return intel_pt_overflow(decoder);  in intel_pt_walk_fup_tip()
1973 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1974 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1975 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1977 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1978 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1980 			decoder->pge = false;  in intel_pt_walk_fup_tip()
1981 			decoder->continuous_period = false;  in intel_pt_walk_fup_tip()
1982 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_fup_tip()
1983 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
1987 			decoder->pge = true;  in intel_pt_walk_fup_tip()
1989 				     decoder->ip);  in intel_pt_walk_fup_tip()
1990 			decoder->state.from_ip = 0;  in intel_pt_walk_fup_tip()
1991 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1992 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1994 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1995 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1997 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_fup_tip()
1998 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_fup_tip()
1999 			intel_pt_set_nr(decoder);  in intel_pt_walk_fup_tip()
2003 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2004 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2005 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2007 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2008 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2010 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
2014 			intel_pt_update_pip(decoder);  in intel_pt_walk_fup_tip()
2018 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2019 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_fup_tip()
2020 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_fup_tip()
2024 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2028 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_fup_tip()
2037 			return intel_pt_bug(decoder);  in intel_pt_walk_fup_tip()
2042 static int intel_pt_resample(struct intel_pt_decoder *decoder)  in intel_pt_resample()  argument
2044 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_resample()
2045 	decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_resample()
2046 	decoder->state.from_ip = decoder->ip;  in intel_pt_resample()
2047 	decoder->state.to_ip = 0;  in intel_pt_resample()
2144 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)  in intel_pt_ovf_fup_lookahead()  argument
2151 	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);  in intel_pt_ovf_fup_lookahead()
2171 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)  in intel_pt_ctc_to_tsc()  argument
2173 	if (decoder->tsc_ctc_mult)  in intel_pt_ctc_to_tsc()
2174 		return ctc * decoder->tsc_ctc_mult;  in intel_pt_ctc_to_tsc()
2176 		return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);  in intel_pt_ctc_to_tsc()
2179 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_expected_tsc()  argument
2202 	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;  in intel_pt_calc_expected_tsc()
2213 		intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));  in intel_pt_calc_expected_tsc()
2220 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_expected_tsc()  argument
2226 	return intel_pt_calc_expected_tsc(decoder, ctc, fc,  in intel_pt_expected_tsc()
2227 					  decoder->ctc_timestamp,  in intel_pt_expected_tsc()
2231 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc()  argument
2234 	uint64_t payload = decoder->packet.payload;  in intel_pt_translate_vm_tsc()
2237 	decoder->packet.payload -= vmcs_info->tsc_offset;  in intel_pt_translate_vm_tsc()
2239 	decoder->packet.payload &= SEVEN_BYTES;  in intel_pt_translate_vm_tsc()
2245 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_translate_vm_tsc()
2246 		memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);  in intel_pt_translate_vm_tsc()
2250 		     payload, decoder->packet.payload, vmcs_info->vmcs,  in intel_pt_translate_vm_tsc()
2254 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc_offset()  argument
2262 	intel_pt_translate_vm_tsc(decoder, &vmcs_info);  in intel_pt_translate_vm_tsc_offset()
2281 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_psb()  argument
2285 	data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_psb()
2286 	data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_psb()
2287 	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);  in intel_pt_vm_tm_corr_psb()
2290 	decoder->in_psb = data->psbend;  in intel_pt_vm_tm_corr_psb()
2293 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_first_tsc()  argument
2298 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_first_tsc()
2306 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2307 				decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2312 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2315 		decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_first_tsc()
2316 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_first_tsc()
2321 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2328 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_tsc()  argument
2343 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2345 		data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_tsc()
2346 		data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_tsc()
2348 		intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);  in intel_pt_vm_tm_corr_tsc()
2349 		if (decoder->pge) {  in intel_pt_vm_tm_corr_tsc()
2351 			data->pip_packet.payload = decoder->pip_payload;  in intel_pt_vm_tm_corr_tsc()
2361 	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;  in intel_pt_vm_tm_corr_tsc()
2365 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_tsc()
2367 	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;  in intel_pt_vm_tm_corr_tsc()
2368 	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);  in intel_pt_vm_tm_corr_tsc()
2370 	if (!decoder->ctc_timestamp) {  in intel_pt_vm_tm_corr_tsc()
2371 		intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);  in intel_pt_vm_tm_corr_tsc()
2375 	expected_tsc = intel_pt_expected_tsc(decoder, data);  in intel_pt_vm_tm_corr_tsc()
2385 				intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2386 				decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2390 			decoder->last_reliable_timestamp = host_tsc;  in intel_pt_vm_tm_corr_tsc()
2391 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_tsc()
2396 		if (decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2419 		if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_tsc()
2421 			decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2432 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&  in intel_pt_vm_tm_corr_tsc()
2433 			   decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2437 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2462 		intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_tsc()
2472 			if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_tsc()
2475 		intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2478 	decoder->vm_tm_corr_reliable = reliable;  in intel_pt_vm_tm_corr_tsc()
2481 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)  in intel_pt_vm_tm_corr_pebs_tsc()  argument
2483 	uint64_t host_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2484 	uint64_t guest_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2488 	vmcs = decoder->vmcs;  in intel_pt_vm_tm_corr_pebs_tsc()
2492 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_pebs_tsc()
2494 	if (decoder->pge) {  in intel_pt_vm_tm_corr_pebs_tsc()
2495 		if (in_vm(decoder->pip_payload)) { /* Guest */  in intel_pt_vm_tm_corr_pebs_tsc()
2504 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_pebs_tsc()
2513 		host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);  in intel_pt_vm_tm_corr_pebs_tsc()
2518 		if (!intel_pt_time_in_range(decoder, host_tsc) &&  in intel_pt_vm_tm_corr_pebs_tsc()
2519 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_vm_tm_corr_pebs_tsc()
2522 		if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_pebs_tsc()
2524 		host_tsc = decoder->timestamp;  in intel_pt_vm_tm_corr_pebs_tsc()
2527 	decoder->packet.payload = host_tsc;  in intel_pt_vm_tm_corr_pebs_tsc()
2529 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_vm_tm_corr_pebs_tsc()
2530 		memcpy((void *)decoder->buf + 1, &host_tsc, 8);  in intel_pt_vm_tm_corr_pebs_tsc()
2533 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)  in intel_pt_vm_time_correlation()  argument
2539 	if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2540 		intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2543 		err = intel_pt_get_next_packet(decoder);  in intel_pt_vm_time_correlation()
2549 		switch (decoder->packet.type) {  in intel_pt_vm_time_correlation()
2551 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2552 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2558 			decoder->pge = true;  in intel_pt_vm_time_correlation()
2562 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2563 			pge = decoder->pge;  in intel_pt_vm_time_correlation()
2564 			decoder->pge = intel_pt_ovf_fup_lookahead(decoder);  in intel_pt_vm_time_correlation()
2565 			if (pge != decoder->pge)  in intel_pt_vm_time_correlation()
2567 			if (!decoder->pge)  in intel_pt_vm_time_correlation()
2568 				decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2572 			if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2573 				decoder->pge = true;  in intel_pt_vm_time_correlation()
2577 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2578 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2579 			decoder->have_tma = false;  in intel_pt_vm_time_correlation()
2583 			intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2587 			decoder->pip_payload = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2591 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2595 			intel_pt_vm_tm_corr_tsc(decoder, &data);  in intel_pt_vm_time_correlation()
2596 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2597 			decoder->vm_tm_corr_same_buf = true;  in intel_pt_vm_time_correlation()
2598 			decoder->vm_tm_corr_continuous = decoder->pge;  in intel_pt_vm_time_correlation()
2602 			intel_pt_calc_tma(decoder);  in intel_pt_vm_time_correlation()
2606 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2610 			intel_pt_calc_cbr(decoder);  in intel_pt_vm_time_correlation()
2614 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2619 			if (decoder->packet.payload != NO_VMCS)  in intel_pt_vm_time_correlation()
2620 				decoder->vmcs = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2624 			decoder->blk_type = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2628 			if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&  in intel_pt_vm_time_correlation()
2629 			    decoder->packet.count == 2)  in intel_pt_vm_time_correlation()
2630 				intel_pt_vm_tm_corr_pebs_tsc(decoder);  in intel_pt_vm_time_correlation()
2635 			decoder->blk_type = 0;  in intel_pt_vm_time_correlation()
2663 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2666 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)  in intel_pt_hop_trace()  argument
2669 	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {  in intel_pt_hop_trace()
2670 		*err = intel_pt_scan_for_psb(decoder);  in intel_pt_hop_trace()
2675 	switch (decoder->packet.type) {  in intel_pt_hop_trace()
2680 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2681 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2684 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2685 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_hop_trace()
2686 		decoder->state.from_ip = 0;  in intel_pt_hop_trace()
2687 		decoder->state.to_ip = decoder->ip;  in intel_pt_hop_trace()
2688 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2692 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2693 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2696 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2697 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2698 		decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2699 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2700 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2704 		if (!decoder->packet.count)  in intel_pt_hop_trace()
2706 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2707 		if (intel_pt_fup_event(decoder))  in intel_pt_hop_trace()
2709 		if (!decoder->branch_enable)  in intel_pt_hop_trace()
2712 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2713 			decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2714 			decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2717 		*err = intel_pt_walk_fup_tip(decoder);  in intel_pt_hop_trace()
2719 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_hop_trace()
2723 		decoder->state.psb_offset = decoder->pos;  in intel_pt_hop_trace()
2724 		decoder->psb_ip = 0;  in intel_pt_hop_trace()
2725 		decoder->last_ip = 0;  in intel_pt_hop_trace()
2726 		decoder->have_last_ip = true;  in intel_pt_hop_trace()
2727 		*err = intel_pt_walk_psbend(decoder);  in intel_pt_hop_trace()
2732 		decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_hop_trace()
2733 		if (decoder->psb_ip) {  in intel_pt_hop_trace()
2734 			decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2735 			decoder->ip = decoder->psb_ip;  in intel_pt_hop_trace()
2737 		decoder->state.from_ip = decoder->psb_ip;  in intel_pt_hop_trace()
2738 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2858 static int intel_pt_psb(struct intel_pt_decoder *decoder)  in intel_pt_psb()  argument
2862 	decoder->last_ip = 0;  in intel_pt_psb()
2863 	decoder->psb_ip = 0;  in intel_pt_psb()
2864 	decoder->have_last_ip = true;  in intel_pt_psb()
2865 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_psb()
2866 	err = intel_pt_walk_psbend(decoder);  in intel_pt_psb()
2869 	decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_psb()
2870 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_psb()
2871 	decoder->state.to_ip = 0;  in intel_pt_psb()
2875 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)  in intel_pt_fup_in_psb()  argument
2879 	if (decoder->ip != decoder->last_ip) {  in intel_pt_fup_in_psb()
2880 		err = intel_pt_walk_fup(decoder);  in intel_pt_fup_in_psb()
2885 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_fup_in_psb()
2886 	err = intel_pt_psb(decoder);  in intel_pt_fup_in_psb()
2888 		decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_fup_in_psb()
2895 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)  in intel_pt_psb_with_fup()  argument
2899 	if (!decoder->branch_enable || !decoder->pge)  in intel_pt_psb_with_fup()
2902 	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);  in intel_pt_psb_with_fup()
2906 	decoder->packet = data.fup_packet;  in intel_pt_psb_with_fup()
2907 	intel_pt_set_last_ip(decoder);  in intel_pt_psb_with_fup()
2908 	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;  in intel_pt_psb_with_fup()
2910 	*err = intel_pt_fup_in_psb(decoder);  in intel_pt_psb_with_fup()
2915 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)  in intel_pt_walk_trace()  argument
2922 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
2926 		if (decoder->cyc_threshold) {  in intel_pt_walk_trace()
2927 			if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)  in intel_pt_walk_trace()
2928 				decoder->sample_cyc = false;  in intel_pt_walk_trace()
2929 			last_packet_type = decoder->packet.type;  in intel_pt_walk_trace()
2932 		if (decoder->hop) {  in intel_pt_walk_trace()
2933 			switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {  in intel_pt_walk_trace()
2945 		switch (decoder->packet.type) {  in intel_pt_walk_trace()
2947 			if (!decoder->packet.count)  in intel_pt_walk_trace()
2949 			decoder->tnt = decoder->packet;  in intel_pt_walk_trace()
2950 			decoder->pkt_state = INTEL_PT_STATE_TNT;  in intel_pt_walk_trace()
2951 			err = intel_pt_walk_tnt(decoder);  in intel_pt_walk_trace()
2957 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
2958 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
2959 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;  in intel_pt_walk_trace()
2960 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
2963 			decoder->pge = true;  in intel_pt_walk_trace()
2964 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_trace()
2965 			intel_pt_set_nr(decoder);  in intel_pt_walk_trace()
2966 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
2968 						decoder->pos);  in intel_pt_walk_trace()
2971 			intel_pt_set_ip(decoder);  in intel_pt_walk_trace()
2972 			decoder->state.from_ip = 0;  in intel_pt_walk_trace()
2973 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_trace()
2974 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_trace()
2979 			if (decoder->hop)  in intel_pt_walk_trace()
2980 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_walk_trace()
2985 			return intel_pt_overflow(decoder);  in intel_pt_walk_trace()
2988 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
2989 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
2990 			decoder->pkt_state = INTEL_PT_STATE_TIP;  in intel_pt_walk_trace()
2991 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
2994 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
2996 						decoder->pos);  in intel_pt_walk_trace()
3000 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3001 			if (!decoder->branch_enable) {  in intel_pt_walk_trace()
3002 				decoder->ip = decoder->last_ip;  in intel_pt_walk_trace()
3003 				if (intel_pt_fup_event(decoder))  in intel_pt_walk_trace()
3008 			if (decoder->set_fup_mwait)  in intel_pt_walk_trace()
3011 				decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;  in intel_pt_walk_trace()
3013 				decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_walk_trace()
3014 			err = intel_pt_walk_fup(decoder);  in intel_pt_walk_trace()
3021 			return intel_pt_walk_fup_tip(decoder);  in intel_pt_walk_trace()
3024 			decoder->pge = false;  in intel_pt_walk_trace()
3025 			decoder->continuous_period = false;  in intel_pt_walk_trace()
3026 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_trace()
3027 			decoder->have_tma = false;  in intel_pt_walk_trace()
3031 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_trace()
3032 			decoder->psb_ip = 0;  in intel_pt_walk_trace()
3033 			if (intel_pt_psb_with_fup(decoder, &err))  in intel_pt_walk_trace()
3035 			err = intel_pt_psb(decoder);  in intel_pt_walk_trace()
3041 			intel_pt_update_pip(decoder);  in intel_pt_walk_trace()
3045 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_trace()
3046 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)  in intel_pt_walk_trace()
3052 			if (!decoder->mtc_insn)  in intel_pt_walk_trace()
3054 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3056 			if (!decoder->timestamp)  in intel_pt_walk_trace()
3058 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_trace()
3059 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3060 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3061 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3065 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_trace()
3069 			intel_pt_calc_tma(decoder);  in intel_pt_walk_trace()
3073 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_trace()
3077 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_trace()
3078 			if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
3079 				decoder->state.type = 0;  in intel_pt_walk_trace()
3085 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_trace()
3090 			if (!decoder->pge || decoder->in_psb) {  in intel_pt_walk_trace()
3091 				intel_pt_update_in_tx(decoder);  in intel_pt_walk_trace()
3094 			err = intel_pt_mode_tsx(decoder, &no_tip);  in intel_pt_walk_trace()
3100 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3109 			decoder->fup_ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3110 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3113 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3114 				decoder->set_fup_ptw = true;  in intel_pt_walk_trace()
3118 						decoder->pos);  in intel_pt_walk_trace()
3123 			decoder->state.type = INTEL_PT_PTW;  in intel_pt_walk_trace()
3124 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3125 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3126 			decoder->state.ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3130 			decoder->fup_mwait_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3131 			decoder->set_fup_mwait = true;  in intel_pt_walk_trace()
3135 			if (decoder->set_fup_mwait) {  in intel_pt_walk_trace()
3136 				decoder->fup_pwre_payload =  in intel_pt_walk_trace()
3137 							decoder->packet.payload;  in intel_pt_walk_trace()
3138 				decoder->set_fup_pwre = true;  in intel_pt_walk_trace()
3141 			decoder->state.type = INTEL_PT_PWR_ENTRY;  in intel_pt_walk_trace()
3142 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3143 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3144 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3148 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3151 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3152 				decoder->set_fup_exstop = true;  in intel_pt_walk_trace()
3156 						decoder->pos);  in intel_pt_walk_trace()
3161 			decoder->state.type = INTEL_PT_EX_STOP;  in intel_pt_walk_trace()
3162 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3163 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3167 			decoder->state.type = INTEL_PT_PWR_EXIT;  in intel_pt_walk_trace()
3168 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3169 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3170 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3174 			intel_pt_bbp(decoder);  in intel_pt_walk_trace()
3178 			intel_pt_bip(decoder);  in intel_pt_walk_trace()
3182 			decoder->state.type = INTEL_PT_BLK_ITEMS;  in intel_pt_walk_trace()
3183 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3184 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3188 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3191 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3192 				decoder->set_fup_bep = true;  in intel_pt_walk_trace()
3196 						decoder->pos);  in intel_pt_walk_trace()
3201 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3206 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)  in intel_pt_have_ip()  argument
3208 	return decoder->packet.count &&  in intel_pt_have_ip()
3209 	       (decoder->have_last_ip || decoder->packet.count == 3 ||  in intel_pt_have_ip()
3210 		decoder->packet.count == 6);  in intel_pt_have_ip()
3214 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)  in intel_pt_walk_psb()  argument
3218 	decoder->in_psb = true;  in intel_pt_walk_psb()
3221 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psb()
3225 		switch (decoder->packet.type) {  in intel_pt_walk_psb()
3227 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3247 			decoder->pge = true;  in intel_pt_walk_psb()
3248 			if (intel_pt_have_ip(decoder)) {  in intel_pt_walk_psb()
3249 				uint64_t current_ip = decoder->ip;  in intel_pt_walk_psb()
3251 				intel_pt_set_ip(decoder);  in intel_pt_walk_psb()
3252 				decoder->psb_ip = decoder->ip;  in intel_pt_walk_psb()
3255 							decoder->ip);  in intel_pt_walk_psb()
3260 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psb()
3264 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psb()
3268 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psb()
3272 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psb()
3276 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psb()
3280 			intel_pt_set_pip(decoder);  in intel_pt_walk_psb()
3284 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psb()
3288 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psb()
3292 			decoder->pge = false;  in intel_pt_walk_psb()
3293 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3294 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_psb()
3298 			decoder->have_tma = false;  in intel_pt_walk_psb()
3300 			if (decoder->ip)  in intel_pt_walk_psb()
3301 				decoder->pkt_state = INTEL_PT_STATE_ERR4;  in intel_pt_walk_psb()
3303 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_psb()
3308 			err = intel_pt_bug(decoder);  in intel_pt_walk_psb()
3312 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psb()
3328 	decoder->in_psb = false;  in intel_pt_walk_psb()
3333 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)  in intel_pt_walk_to_ip()  argument
3338 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_to_ip()
3342 		switch (decoder->packet.type) {  in intel_pt_walk_to_ip()
3344 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3345 			decoder->pge = false;  in intel_pt_walk_to_ip()
3346 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3347 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3348 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3350 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_to_ip()
3354 			decoder->pge = true;  in intel_pt_walk_to_ip()
3355 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_to_ip()
3356 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3357 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3358 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3360 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_to_ip()
3364 			decoder->pge = true;  in intel_pt_walk_to_ip()
3365 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3366 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3367 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3372 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3373 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3374 			if (decoder->ip)  in intel_pt_walk_to_ip()
3379 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_to_ip()
3383 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_to_ip()
3387 			intel_pt_calc_tma(decoder);  in intel_pt_walk_to_ip()
3391 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_to_ip()
3395 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_to_ip()
3399 			intel_pt_set_pip(decoder);  in intel_pt_walk_to_ip()
3403 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_to_ip()
3407 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_to_ip()
3411 			return intel_pt_overflow(decoder);  in intel_pt_walk_to_ip()
3414 			return intel_pt_bug(decoder);  in intel_pt_walk_to_ip()
3417 			decoder->pge = false;  in intel_pt_walk_to_ip()
3418 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3419 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_to_ip()
3420 			decoder->have_tma = false;  in intel_pt_walk_to_ip()
3424 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_to_ip()
3425 			decoder->psb_ip = 0;  in intel_pt_walk_to_ip()
3426 			decoder->last_ip = 0;  in intel_pt_walk_to_ip()
3427 			decoder->have_last_ip = true;  in intel_pt_walk_to_ip()
3428 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_to_ip()
3429 			err = intel_pt_walk_psb(decoder);  in intel_pt_walk_to_ip()
3432 			decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_walk_to_ip()
3433 			decoder->state.from_ip = decoder->psb_ip;  in intel_pt_walk_to_ip()
3434 			decoder->state.to_ip = 0;  in intel_pt_walk_to_ip()
3459 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)  in intel_pt_sync_ip()  argument
3463 	decoder->set_fup_tx_flags = false;  in intel_pt_sync_ip()
3464 	decoder->set_fup_ptw = false;  in intel_pt_sync_ip()
3465 	decoder->set_fup_mwait = false;  in intel_pt_sync_ip()
3466 	decoder->set_fup_pwre = false;  in intel_pt_sync_ip()
3467 	decoder->set_fup_exstop = false;  in intel_pt_sync_ip()
3468 	decoder->set_fup_bep = false;  in intel_pt_sync_ip()
3470 	if (!decoder->branch_enable) {  in intel_pt_sync_ip()
3471 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3472 		decoder->overflow = false;  in intel_pt_sync_ip()
3473 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync_ip()
3478 	err = intel_pt_walk_to_ip(decoder);  in intel_pt_sync_ip()
3479 	if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))  in intel_pt_sync_ip()
3483 	if (decoder->hop)  in intel_pt_sync_ip()
3484 		decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync_ip()
3486 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3487 	decoder->overflow = false;  in intel_pt_sync_ip()
3489 	decoder->state.from_ip = 0;  in intel_pt_sync_ip()
3490 	decoder->state.to_ip = decoder->ip;  in intel_pt_sync_ip()
3491 	intel_pt_log_to("Setting IP", decoder->ip);  in intel_pt_sync_ip()
3496 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)  in intel_pt_part_psb()  argument
3498 	const unsigned char *end = decoder->buf + decoder->len;  in intel_pt_part_psb()
3502 		if (i > decoder->len)  in intel_pt_part_psb()
3510 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)  in intel_pt_rest_psb()  argument
3515 	if (rest_psb > decoder->len ||  in intel_pt_rest_psb()
3516 	    memcmp(decoder->buf, psb + part_psb, rest_psb))  in intel_pt_rest_psb()
3522 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,  in intel_pt_get_split_psb()  argument
3527 	decoder->pos += decoder->len;  in intel_pt_get_split_psb()
3528 	decoder->len = 0;  in intel_pt_get_split_psb()
3530 	ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_split_psb()
3534 	rest_psb = intel_pt_rest_psb(decoder, part_psb);  in intel_pt_get_split_psb()
3538 	decoder->pos -= part_psb;  in intel_pt_get_split_psb()
3539 	decoder->next_buf = decoder->buf + rest_psb;  in intel_pt_get_split_psb()
3540 	decoder->next_len = decoder->len - rest_psb;  in intel_pt_get_split_psb()
3541 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);  in intel_pt_get_split_psb()
3542 	decoder->buf = decoder->temp_buf;  in intel_pt_get_split_psb()
3543 	decoder->len = INTEL_PT_PSB_LEN;  in intel_pt_get_split_psb()
3548 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)  in intel_pt_scan_for_psb()  argument
3555 		if (!decoder->len) {  in intel_pt_scan_for_psb()
3556 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_scan_for_psb()
3561 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,  in intel_pt_scan_for_psb()
3566 			part_psb = intel_pt_part_psb(decoder);  in intel_pt_scan_for_psb()
3568 				ret = intel_pt_get_split_psb(decoder, part_psb);  in intel_pt_scan_for_psb()
3572 				decoder->pos += decoder->len;  in intel_pt_scan_for_psb()
3573 				decoder->len = 0;  in intel_pt_scan_for_psb()
3578 		decoder->pkt_step = next - decoder->buf;  in intel_pt_scan_for_psb()
3579 		return intel_pt_get_next_packet(decoder);  in intel_pt_scan_for_psb()
3583 static int intel_pt_sync(struct intel_pt_decoder *decoder)  in intel_pt_sync()  argument
3587 	decoder->pge = false;  in intel_pt_sync()
3588 	decoder->continuous_period = false;  in intel_pt_sync()
3589 	decoder->have_last_ip = false;  in intel_pt_sync()
3590 	decoder->last_ip = 0;  in intel_pt_sync()
3591 	decoder->psb_ip = 0;  in intel_pt_sync()
3592 	decoder->ip = 0;  in intel_pt_sync()
3593 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_sync()
3595 	err = intel_pt_scan_for_psb(decoder);  in intel_pt_sync()
3599 	if (decoder->vm_time_correlation) {  in intel_pt_sync()
3600 		decoder->in_psb = true;  in intel_pt_sync()
3601 		if (!decoder->timestamp)  in intel_pt_sync()
3602 			decoder->timestamp = 1;  in intel_pt_sync()
3603 		decoder->state.type = 0;  in intel_pt_sync()
3604 		decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;  in intel_pt_sync()
3608 	decoder->have_last_ip = true;  in intel_pt_sync()
3609 	decoder->pkt_state = INTEL_PT_STATE_NO_IP;  in intel_pt_sync()
3611 	err = intel_pt_walk_psb(decoder);  in intel_pt_sync()
3615 	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */  in intel_pt_sync()
3616 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_sync()
3617 	decoder->state.to_ip = 0;  in intel_pt_sync()
3619 	if (decoder->ip) {  in intel_pt_sync()
3624 		if (decoder->hop)  in intel_pt_sync()
3625 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync()
3627 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
3633 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_est_timestamp()  argument
3635 	uint64_t est = decoder->sample_insn_cnt << 1;  in intel_pt_est_timestamp()
3637 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)  in intel_pt_est_timestamp()
3640 	est *= decoder->max_non_turbo_ratio;  in intel_pt_est_timestamp()
3641 	est /= decoder->cbr;  in intel_pt_est_timestamp()
3643 	return decoder->sample_timestamp + est;  in intel_pt_est_timestamp()
3646 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)  in intel_pt_decode()  argument
3651 		decoder->state.type = INTEL_PT_BRANCH;  in intel_pt_decode()
3652 		decoder->state.flags = 0;  in intel_pt_decode()
3654 		switch (decoder->pkt_state) {  in intel_pt_decode()
3656 			err = intel_pt_sync(decoder);  in intel_pt_decode()
3659 			decoder->have_last_ip = false;  in intel_pt_decode()
3660 			decoder->last_ip = 0;  in intel_pt_decode()
3661 			decoder->ip = 0;  in intel_pt_decode()
3664 			err = intel_pt_sync_ip(decoder);  in intel_pt_decode()
3667 			err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3671 			err = intel_pt_walk_tnt(decoder);  in intel_pt_decode()
3673 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3677 			err = intel_pt_walk_tip(decoder);  in intel_pt_decode()
3680 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
3682 				err = intel_pt_walk_fup_tip(decoder);  in intel_pt_decode()
3685 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
3687 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3690 			err = intel_pt_fup_in_psb(decoder);  in intel_pt_decode()
3693 			err = intel_pt_resample(decoder);  in intel_pt_decode()
3696 			err = intel_pt_vm_time_correlation(decoder);  in intel_pt_decode()
3699 			err = intel_pt_bug(decoder);  in intel_pt_decode()
3705 		decoder->state.err = intel_pt_ext_err(err);  in intel_pt_decode()
3706 		decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
3707 		intel_pt_update_sample_time(decoder);  in intel_pt_decode()
3708 		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
3709 		intel_pt_set_nr(decoder);  in intel_pt_decode()
3711 		decoder->state.err = 0;  in intel_pt_decode()
3712 		if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_decode()
3713 			decoder->cbr_seen = decoder->cbr;  in intel_pt_decode()
3714 			if (!decoder->state.type) {  in intel_pt_decode()
3715 				decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
3716 				decoder->state.to_ip = 0;  in intel_pt_decode()
3718 			decoder->state.type |= INTEL_PT_CBR_CHG;  in intel_pt_decode()
3719 			decoder->state.cbr_payload = decoder->cbr_payload;  in intel_pt_decode()
3720 			decoder->state.cbr = decoder->cbr;  in intel_pt_decode()
3722 		if (intel_pt_sample_time(decoder->pkt_state)) {  in intel_pt_decode()
3723 			intel_pt_update_sample_time(decoder);  in intel_pt_decode()
3724 			if (decoder->sample_cyc) {  in intel_pt_decode()
3725 				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
3726 				decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
3727 				decoder->sample_cyc = false;  in intel_pt_decode()
3734 		if (!decoder->have_cyc)  in intel_pt_decode()
3735 			decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
3739 	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)  in intel_pt_decode()
3740 		decoder->sample_timestamp = decoder->tsc_timestamp;  in intel_pt_decode()
3742 	decoder->state.from_nr = decoder->nr;  in intel_pt_decode()
3743 	decoder->state.to_nr = decoder->next_nr;  in intel_pt_decode()
3744 	decoder->nr = decoder->next_nr;  in intel_pt_decode()
3746 	decoder->state.timestamp = decoder->sample_timestamp;  in intel_pt_decode()
3747 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);  in intel_pt_decode()
3748 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;  in intel_pt_decode()
3749 	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;  in intel_pt_decode()
3751 	return &decoder->state;  in intel_pt_decode()
4127 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)  in intel_pt_fast_forward()  argument
4137 	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);  in intel_pt_fast_forward()
4144 			decoder->pos += decoder->len;  in intel_pt_fast_forward()
4145 			decoder->len = 0;  in intel_pt_fast_forward()
4146 			err = intel_pt_get_next_data(decoder, true);  in intel_pt_fast_forward()
4150 		} while (decoder->buf_timestamp != d.buf_timestamp);  in intel_pt_fast_forward()
4153 	if (!decoder->buf)  in intel_pt_fast_forward()
4156 	buf = (unsigned char *)decoder->buf;  in intel_pt_fast_forward()
4157 	len = decoder->len;  in intel_pt_fast_forward()
4172 		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);  in intel_pt_fast_forward()
4181 			decoder->pos += decoder->len - len;  in intel_pt_fast_forward()
4182 			decoder->buf = buf;  in intel_pt_fast_forward()
4183 			decoder->len = len;  in intel_pt_fast_forward()
4184 			intel_pt_reposition(decoder);  in intel_pt_fast_forward()