22 #include "intel-pt-insn-decoder.h"
23 #include "intel-pt-pkt-decoder.h"
24 #include "intel-pt-decoder.h"
111 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
269 static bool intel_pt_print_once(struct intel_pt_decoder *decoder,  in intel_pt_print_once()  argument
274 	if (decoder->print_once & bit)  in intel_pt_print_once()
276 	decoder->print_once |= bit;  in intel_pt_print_once()
288 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)  in intel_pt_setup_period()  argument
290 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {  in intel_pt_setup_period()
293 		period = intel_pt_lower_power_of_2(decoder->period);  in intel_pt_setup_period()
294 		decoder->period_mask  = ~(period - 1);  in intel_pt_setup_period()
295 		decoder->period_ticks = period;  in intel_pt_setup_period()
308 	struct intel_pt_decoder *decoder;  in intel_pt_decoder_new()  local
313 	decoder = zalloc(sizeof(struct intel_pt_decoder));  in intel_pt_decoder_new()
314 	if (!decoder)  in intel_pt_decoder_new()
317 	decoder->get_trace          = params->get_trace;  in intel_pt_decoder_new()
318 	decoder->walk_insn          = params->walk_insn;  in intel_pt_decoder_new()
319 	decoder->pgd_ip             = params->pgd_ip;  in intel_pt_decoder_new()
320 	decoder->lookahead          = params->lookahead;  in intel_pt_decoder_new()
321 	decoder->findnew_vmcs_info  = params->findnew_vmcs_info;  in intel_pt_decoder_new()
322 	decoder->data               = params->data;  in intel_pt_decoder_new()
323 	decoder->return_compression = params->return_compression;  in intel_pt_decoder_new()
324 	decoder->branch_enable      = params->branch_enable;  in intel_pt_decoder_new()
325 	decoder->hop                = params->quick >= 1;  in intel_pt_decoder_new()
326 	decoder->leap               = params->quick >= 2;  in intel_pt_decoder_new()
327 	decoder->vm_time_correlation = params->vm_time_correlation;  in intel_pt_decoder_new()
328 	decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run;  in intel_pt_decoder_new()
329 	decoder->first_timestamp    = params->first_timestamp;  in intel_pt_decoder_new()
330 	decoder->last_reliable_timestamp = params->first_timestamp;  in intel_pt_decoder_new()
331 	decoder->max_loops          = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS;  in intel_pt_decoder_new()
333 	decoder->flags              = params->flags;  in intel_pt_decoder_new()
335 	decoder->ctl                = params->ctl;  in intel_pt_decoder_new()
336 	decoder->period             = params->period;  in intel_pt_decoder_new()
337 	decoder->period_type        = params->period_type;  in intel_pt_decoder_new()
339 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
340 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
342 	decoder->cyc_threshold = intel_pt_cyc_threshold(decoder->ctl);  in intel_pt_decoder_new()
344 	intel_pt_setup_period(decoder);  in intel_pt_decoder_new()
346 	decoder->mtc_shift = params->mtc_period;  in intel_pt_decoder_new()
347 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;  in intel_pt_decoder_new()
349 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;  in intel_pt_decoder_new()
350 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
352 	if (!decoder->tsc_ctc_ratio_n)  in intel_pt_decoder_new()
353 		decoder->tsc_ctc_ratio_d = 0;  in intel_pt_decoder_new()
355 	if (decoder->tsc_ctc_ratio_d) {  in intel_pt_decoder_new()
356 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))  in intel_pt_decoder_new()
357 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /  in intel_pt_decoder_new()
358 						decoder->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
367 	decoder->tsc_slip = 0x10000;  in intel_pt_decoder_new()
369 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);  in intel_pt_decoder_new()
370 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);  in intel_pt_decoder_new()
371 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
372 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);  in intel_pt_decoder_new()
373 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);  in intel_pt_decoder_new()
375 	if (decoder->hop)  in intel_pt_decoder_new()
378 	return decoder;  in intel_pt_decoder_new()
381 void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder,  in intel_pt_set_first_timestamp()  argument
384 	decoder->first_timestamp = first_timestamp;  in intel_pt_set_first_timestamp()
458 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)  in intel_pt_decoder_free()  argument
460 	intel_pt_free_stack(&decoder->stack);  in intel_pt_decoder_free()
461 	free(decoder);  in intel_pt_decoder_free()
548 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_last_ip()  argument
550 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);  in intel_pt_set_last_ip()
551 	decoder->have_last_ip = true;  in intel_pt_set_last_ip()
554 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_ip()  argument
556 	intel_pt_set_last_ip(decoder);  in intel_pt_set_ip()
557 	decoder->ip = decoder->last_ip;  in intel_pt_set_ip()
560 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)  in intel_pt_decoder_log_packet()  argument
562 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,  in intel_pt_decoder_log_packet()
563 			    decoder->buf);  in intel_pt_decoder_log_packet()
566 static int intel_pt_bug(struct intel_pt_decoder *decoder)  in intel_pt_bug()  argument
569 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_bug()
573 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)  in intel_pt_clear_tx_flags()  argument
575 	decoder->tx_flags = 0;  in intel_pt_clear_tx_flags()
578 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)  in intel_pt_update_in_tx()  argument
580 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;  in intel_pt_update_in_tx()
583 static inline void intel_pt_update_pip(struct intel_pt_decoder *decoder)  in intel_pt_update_pip()  argument
585 	decoder->pip_payload = decoder->packet.payload;  in intel_pt_update_pip()
588 static inline void intel_pt_update_nr(struct intel_pt_decoder *decoder)  in intel_pt_update_nr()  argument
590 	decoder->next_nr = decoder->pip_payload & 1;  in intel_pt_update_nr()
593 static inline void intel_pt_set_nr(struct intel_pt_decoder *decoder)  in intel_pt_set_nr()  argument
595 	decoder->nr = decoder->pip_payload & 1;  in intel_pt_set_nr()
596 	decoder->next_nr = decoder->nr;  in intel_pt_set_nr()
599 static inline void intel_pt_set_pip(struct intel_pt_decoder *decoder)  in intel_pt_set_pip()  argument
601 	intel_pt_update_pip(decoder);  in intel_pt_set_pip()
602 	intel_pt_set_nr(decoder);  in intel_pt_set_pip()
605 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)  in intel_pt_bad_packet()  argument
607 	intel_pt_clear_tx_flags(decoder);  in intel_pt_bad_packet()
608 	decoder->have_tma = false;  in intel_pt_bad_packet()
609 	decoder->pkt_len = 1;  in intel_pt_bad_packet()
610 	decoder->pkt_step = 1;  in intel_pt_bad_packet()
611 	intel_pt_decoder_log_packet(decoder);  in intel_pt_bad_packet()
612 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {  in intel_pt_bad_packet()
614 		decoder->pkt_state = INTEL_PT_STATE_ERR1;  in intel_pt_bad_packet()
619 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)  in intel_pt_update_sample_time()  argument
621 	decoder->sample_timestamp = decoder->timestamp;  in intel_pt_update_sample_time()
622 	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_update_sample_time()
623 	decoder->state.cycles = decoder->tot_cyc_cnt;  in intel_pt_update_sample_time()
626 static void intel_pt_reposition(struct intel_pt_decoder *decoder)  in intel_pt_reposition()  argument
628 	decoder->ip = 0;  in intel_pt_reposition()
629 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_reposition()
630 	decoder->timestamp = 0;  in intel_pt_reposition()
631 	decoder->have_tma = false;  in intel_pt_reposition()
634 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)  in intel_pt_get_data()  argument
639 	decoder->pkt_step = 0;  in intel_pt_get_data()
642 	ret = decoder->get_trace(&buffer, decoder->data);  in intel_pt_get_data()
645 	decoder->buf = buffer.buf;  in intel_pt_get_data()
646 	decoder->len = buffer.len;  in intel_pt_get_data()
647 	if (!decoder->len) {  in intel_pt_get_data()
651 	decoder->buf_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
653 		intel_pt_reposition(decoder);  in intel_pt_get_data()
654 		decoder->ref_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
655 		decoder->state.trace_nr = buffer.trace_nr;  in intel_pt_get_data()
656 		decoder->vm_tm_corr_same_buf = false;  in intel_pt_get_data()
658 			     decoder->ref_timestamp);  in intel_pt_get_data()
665 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,  in intel_pt_get_next_data()  argument
668 	if (!decoder->next_buf)  in intel_pt_get_next_data()
669 		return intel_pt_get_data(decoder, reposition);  in intel_pt_get_next_data()
671 	decoder->buf = decoder->next_buf;  in intel_pt_get_next_data()
672 	decoder->len = decoder->next_len;  in intel_pt_get_next_data()
673 	decoder->next_buf = 0;  in intel_pt_get_next_data()
674 	decoder->next_len = 0;  in intel_pt_get_next_data()
678 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_split_packet()  argument
680 	unsigned char *buf = decoder->temp_buf;  in intel_pt_get_split_packet()
684 	old_len = decoder->len;  in intel_pt_get_split_packet()
685 	len = decoder->len;  in intel_pt_get_split_packet()
686 	memcpy(buf, decoder->buf, len);  in intel_pt_get_split_packet()
688 	ret = intel_pt_get_data(decoder, false);  in intel_pt_get_split_packet()
690 		decoder->pos += old_len;  in intel_pt_get_split_packet()
695 	if (n > decoder->len)  in intel_pt_get_split_packet()
696 		n = decoder->len;  in intel_pt_get_split_packet()
697 	memcpy(buf + len, decoder->buf, n);  in intel_pt_get_split_packet()
700 	decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_split_packet()
701 	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_split_packet()
703 		decoder->next_buf = decoder->buf;  in intel_pt_get_split_packet()
704 		decoder->next_len = decoder->len;  in intel_pt_get_split_packet()
705 		decoder->buf = buf;  in intel_pt_get_split_packet()
706 		decoder->len = old_len;  in intel_pt_get_split_packet()
707 		return intel_pt_bad_packet(decoder);  in intel_pt_get_split_packet()
710 	decoder->next_buf = decoder->buf + (ret - old_len);  in intel_pt_get_split_packet()
711 	decoder->next_len = decoder->len - (ret - old_len);  in intel_pt_get_split_packet()
713 	decoder->buf = buf;  in intel_pt_get_split_packet()
714 	decoder->len = ret;  in intel_pt_get_split_packet()
720 	struct intel_pt_decoder	  *decoder;  member
731 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,  in intel_pt_pkt_lookahead()  argument
735 	const unsigned char *buf = decoder->buf;  in intel_pt_pkt_lookahead()
736 	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;  in intel_pt_pkt_lookahead()
737 	size_t len = decoder->len;  in intel_pt_pkt_lookahead()
740 	pkt_info.decoder          = decoder;  in intel_pt_pkt_lookahead()
741 	pkt_info.pos              = decoder->pos;  in intel_pt_pkt_lookahead()
742 	pkt_info.pkt_len          = decoder->pkt_step;  in intel_pt_pkt_lookahead()
743 	pkt_info.last_packet_type = decoder->last_packet_type;  in intel_pt_pkt_lookahead()
809 	struct intel_pt_decoder *decoder = pkt_info->decoder;  in intel_pt_calc_cyc_cb()  local
845 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {  in intel_pt_calc_cyc_cb()
847 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_cyc_cb()
854 		data->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_cyc_cb()
857 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
859 				data->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
863 					decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
864 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
887 		    data->timestamp - timestamp < decoder->tsc_slip)  in intel_pt_calc_cyc_cb()
904 		if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_cyc_cb()
909 		ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_cyc_cb()
911 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_cyc_cb()
914 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
915 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
918 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
919 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
937 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cyc_cb()
953 	if (!data->cbr && decoder->cbr) {  in intel_pt_calc_cyc_cb()
954 		data->cbr = decoder->cbr;  in intel_pt_calc_cyc_cb()
955 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_cb()
961 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;  in intel_pt_calc_cyc_cb()
970 	decoder->calc_cyc_to_tsc = cyc_to_tsc;  in intel_pt_calc_cyc_cb()
971 	decoder->have_calc_cyc_to_tsc = true;  in intel_pt_calc_cyc_cb()
984 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_cyc_to_tsc()  argument
990 		.last_mtc       = decoder->last_mtc,  in intel_pt_calc_cyc_to_tsc()
991 		.ctc_timestamp  = decoder->ctc_timestamp,  in intel_pt_calc_cyc_to_tsc()
992 		.ctc_delta      = decoder->ctc_delta,  in intel_pt_calc_cyc_to_tsc()
993 		.tsc_timestamp  = decoder->tsc_timestamp,  in intel_pt_calc_cyc_to_tsc()
994 		.timestamp      = decoder->timestamp,  in intel_pt_calc_cyc_to_tsc()
995 		.have_tma       = decoder->have_tma,  in intel_pt_calc_cyc_to_tsc()
996 		.fixup_last_mtc = decoder->fixup_last_mtc,  in intel_pt_calc_cyc_to_tsc()
1009 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);  in intel_pt_calc_cyc_to_tsc()
1012 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_next_packet()  argument
1016 	decoder->last_packet_type = decoder->packet.type;  in intel_pt_get_next_packet()
1019 		decoder->pos += decoder->pkt_step;  in intel_pt_get_next_packet()
1020 		decoder->buf += decoder->pkt_step;  in intel_pt_get_next_packet()
1021 		decoder->len -= decoder->pkt_step;  in intel_pt_get_next_packet()
1023 		if (!decoder->len) {  in intel_pt_get_next_packet()
1024 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_next_packet()
1029 		decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_next_packet()
1030 		ret = intel_pt_get_packet(decoder->buf, decoder->len,  in intel_pt_get_next_packet()
1031 					  &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_next_packet()
1033 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {  in intel_pt_get_next_packet()
1034 			ret = intel_pt_get_split_packet(decoder);  in intel_pt_get_next_packet()
1039 			return intel_pt_bad_packet(decoder);  in intel_pt_get_next_packet()
1041 		decoder->pkt_len = ret;  in intel_pt_get_next_packet()
1042 		decoder->pkt_step = ret;  in intel_pt_get_next_packet()
1043 		intel_pt_decoder_log_packet(decoder);  in intel_pt_get_next_packet()
1044 	} while (decoder->packet.type == INTEL_PT_PAD);  in intel_pt_get_next_packet()
1049 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)  in intel_pt_next_period()  argument
1053 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_next_period()
1054 	masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1055 	if (decoder->continuous_period) {  in intel_pt_next_period()
1056 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_next_period()
1060 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
1061 		if (masked_timestamp > decoder->last_masked_timestamp) {  in intel_pt_next_period()
1062 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_next_period()
1063 			decoder->continuous_period = true;  in intel_pt_next_period()
1067 	if (masked_timestamp < decoder->last_masked_timestamp)  in intel_pt_next_period()
1068 		return decoder->period_ticks;  in intel_pt_next_period()
1070 	return decoder->period_ticks - (timestamp - masked_timestamp);  in intel_pt_next_period()
1073 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)  in intel_pt_next_sample()  argument
1075 	switch (decoder->period_type) {  in intel_pt_next_sample()
1077 		return decoder->period - decoder->period_insn_cnt;  in intel_pt_next_sample()
1079 		return intel_pt_next_period(decoder);  in intel_pt_next_sample()
1087 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_insn()  argument
1091 	switch (decoder->period_type) {  in intel_pt_sample_insn()
1093 		decoder->period_insn_cnt = 0;  in intel_pt_sample_insn()
1096 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_sample_insn()
1097 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_sample_insn()
1098 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_sample_insn()
1099 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_sample_insn()
1101 			decoder->last_masked_timestamp += decoder->period_ticks;  in intel_pt_sample_insn()
1109 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_insn()
1116 static void intel_pt_sample_fup_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_fup_insn()  argument
1122 	decoder->state.insn_op = INTEL_PT_OP_OTHER;  in intel_pt_sample_fup_insn()
1123 	decoder->state.insn_len = 0;  in intel_pt_sample_fup_insn()
1125 	if (!decoder->branch_enable || !decoder->pge || decoder->hop ||  in intel_pt_sample_fup_insn()
1126 	    decoder->ip != decoder->last_ip)  in intel_pt_sample_fup_insn()
1129 	if (!decoder->mtc_insn)  in intel_pt_sample_fup_insn()
1130 		decoder->mtc_insn = true;  in intel_pt_sample_fup_insn()
1132 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_sample_fup_insn()
1136 	err = decoder->walk_insn(&intel_pt_insn, &insn_cnt, &decoder->ip,  in intel_pt_sample_fup_insn()
1137 				 0, max_insn_cnt, decoder->data);  in intel_pt_sample_fup_insn()
1143 		intel_pt_log_at("ERROR: Unexpected branch at FUP instruction", decoder->ip);  in intel_pt_sample_fup_insn()
1147 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1148 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1149 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1150 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_sample_fup_insn()
1152 	intel_pt_sample_insn(decoder);  in intel_pt_sample_fup_insn()
1154 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_fup_insn()
1155 	decoder->ip += intel_pt_insn.length;  in intel_pt_sample_fup_insn()
1158 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,  in intel_pt_walk_insn()  argument
1164 	if (!decoder->mtc_insn)  in intel_pt_walk_insn()
1165 		decoder->mtc_insn = true;  in intel_pt_walk_insn()
1167 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_walk_insn()
1169 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,  in intel_pt_walk_insn()
1170 				 max_insn_cnt, decoder->data);  in intel_pt_walk_insn()
1172 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1173 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1174 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1175 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1178 		decoder->no_progress = 0;  in intel_pt_walk_insn()
1179 		decoder->pkt_state = INTEL_PT_STATE_ERR2;  in intel_pt_walk_insn()
1181 				decoder->ip);  in intel_pt_walk_insn()
1187 	if (ip && decoder->ip == ip) {  in intel_pt_walk_insn()
1193 		intel_pt_sample_insn(decoder);  in intel_pt_walk_insn()
1196 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_insn()
1197 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1198 		decoder->state.to_ip = 0;  in intel_pt_walk_insn()
1199 		decoder->ip += intel_pt_insn->length;  in intel_pt_walk_insn()
1208 			err = intel_pt_push(&decoder->stack, decoder->ip +  in intel_pt_walk_insn()
1214 		decoder->ret_addr = intel_pt_pop(&decoder->stack);  in intel_pt_walk_insn()
1218 		int cnt = decoder->no_progress++;  in intel_pt_walk_insn()
1220 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1221 		decoder->ip += intel_pt_insn->length +  in intel_pt_walk_insn()
1223 		decoder->state.to_ip = decoder->ip;  in intel_pt_walk_insn()
1228 		 * decoder error results in the decoder erroneously setting the  in intel_pt_walk_insn()
1235 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1236 				decoder->stuck_ip_prd = 1;  in intel_pt_walk_insn()
1237 				decoder->stuck_ip_cnt = 1;  in intel_pt_walk_insn()
1238 			} else if (cnt > decoder->max_loops ||  in intel_pt_walk_insn()
1239 				   decoder->state.to_ip == decoder->stuck_ip) {  in intel_pt_walk_insn()
1241 						decoder->state.to_ip);  in intel_pt_walk_insn()
1242 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_insn()
1245 			} else if (!--decoder->stuck_ip_cnt) {  in intel_pt_walk_insn()
1246 				decoder->stuck_ip_prd += 1;  in intel_pt_walk_insn()
1247 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;  in intel_pt_walk_insn()
1248 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1254 	decoder->no_progress = 0;  in intel_pt_walk_insn()
1256 	decoder->state.insn_op = intel_pt_insn->op;  in intel_pt_walk_insn()
1257 	decoder->state.insn_len = intel_pt_insn->length;  in intel_pt_walk_insn()
1258 	memcpy(decoder->state.insn, intel_pt_insn->buf,  in intel_pt_walk_insn()
1261 	if (decoder->tx_flags & INTEL_PT_IN_TX)  in intel_pt_walk_insn()
1262 		decoder->state.flags |= INTEL_PT_IN_TX;  in intel_pt_walk_insn()
1267 static void intel_pt_mode_exec_status(struct intel_pt_decoder *decoder)  in intel_pt_mode_exec_status()  argument
1269 	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;  in intel_pt_mode_exec_status()
1271 	decoder->exec_mode = decoder->packet.payload;  in intel_pt_mode_exec_status()
1272 	decoder->iflag = iflag;  in intel_pt_mode_exec_status()
1273 	decoder->next_iflag = iflag;  in intel_pt_mode_exec_status()
1274 	decoder->state.from_iflag = iflag;  in intel_pt_mode_exec_status()
1275 	decoder->state.to_iflag = iflag;  in intel_pt_mode_exec_status()
1278 static void intel_pt_mode_exec(struct intel_pt_decoder *decoder)  in intel_pt_mode_exec()  argument
1280 	bool iflag = decoder->packet.count & INTEL_PT_IFLAG;  in intel_pt_mode_exec()
1282 	decoder->exec_mode = decoder->packet.payload;  in intel_pt_mode_exec()
1283 	decoder->next_iflag = iflag;  in intel_pt_mode_exec()
1286 static void intel_pt_sample_iflag(struct intel_pt_decoder *decoder)  in intel_pt_sample_iflag()  argument
1288 	decoder->state.type |= INTEL_PT_IFLAG_CHG;  in intel_pt_sample_iflag()
1289 	decoder->state.from_iflag = decoder->iflag;  in intel_pt_sample_iflag()
1290 	decoder->state.to_iflag = decoder->next_iflag;  in intel_pt_sample_iflag()
1291 	decoder->iflag = decoder->next_iflag;  in intel_pt_sample_iflag()
1294 static void intel_pt_sample_iflag_chg(struct intel_pt_decoder *decoder)  in intel_pt_sample_iflag_chg()  argument
1296 	if (decoder->iflag != decoder->next_iflag)  in intel_pt_sample_iflag_chg()
1297 		intel_pt_sample_iflag(decoder);  in intel_pt_sample_iflag_chg()
1300 static void intel_pt_clear_fup_event(struct intel_pt_decoder *decoder)  in intel_pt_clear_fup_event()  argument
1302 	decoder->set_fup_tx_flags = false;  in intel_pt_clear_fup_event()
1303 	decoder->set_fup_ptw = false;  in intel_pt_clear_fup_event()
1304 	decoder->set_fup_mwait = false;  in intel_pt_clear_fup_event()
1305 	decoder->set_fup_pwre = false;  in intel_pt_clear_fup_event()
1306 	decoder->set_fup_exstop = false;  in intel_pt_clear_fup_event()
1307 	decoder->set_fup_bep = false;  in intel_pt_clear_fup_event()
1308 	decoder->set_fup_cfe_ip = false;  in intel_pt_clear_fup_event()
1309 	decoder->set_fup_cfe = false;  in intel_pt_clear_fup_event()
1310 	decoder->evd_cnt = 0;  in intel_pt_clear_fup_event()
1311 	decoder->set_fup_mode_exec = false;  in intel_pt_clear_fup_event()
1312 	decoder->iflag = decoder->next_iflag;  in intel_pt_clear_fup_event()
1315 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder, bool no_tip)  in intel_pt_fup_event()  argument
1317 	enum intel_pt_sample_type type = decoder->state.type;  in intel_pt_fup_event()
1321 	decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1323 	if (decoder->set_fup_cfe_ip || decoder->set_fup_cfe) {  in intel_pt_fup_event()
1324 		bool ip = decoder->set_fup_cfe_ip;  in intel_pt_fup_event()
1326 		decoder->set_fup_cfe_ip = false;  in intel_pt_fup_event()
1327 		decoder->set_fup_cfe = false;  in intel_pt_fup_event()
1328 		decoder->state.type |= INTEL_PT_EVT;  in intel_pt_fup_event()
1329 		if (!ip && decoder->pge)  in intel_pt_fup_event()
1330 			decoder->state.type |= INTEL_PT_BRANCH;  in intel_pt_fup_event()
1331 		decoder->state.cfe_type = decoder->fup_cfe_pkt.count;  in intel_pt_fup_event()
1332 		decoder->state.cfe_vector = decoder->fup_cfe_pkt.payload;  in intel_pt_fup_event()
1333 		decoder->state.evd_cnt = decoder->evd_cnt;  in intel_pt_fup_event()
1334 		decoder->state.evd = decoder->evd;  in intel_pt_fup_event()
1335 		decoder->evd_cnt = 0;  in intel_pt_fup_event()
1336 		if (ip || decoder->pge)  in intel_pt_fup_event()
1337 			decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1340 	if (decoder->set_fup_mode_exec) {  in intel_pt_fup_event()
1341 		decoder->set_fup_mode_exec = false;  in intel_pt_fup_event()
1342 		intel_pt_sample_iflag(decoder);  in intel_pt_fup_event()
1346 	if (decoder->set_fup_tx_flags) {  in intel_pt_fup_event()
1347 		decoder->set_fup_tx_flags = false;  in intel_pt_fup_event()
1348 		decoder->tx_flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1349 		decoder->state.type |= INTEL_PT_TRANSACTION;  in intel_pt_fup_event()
1350 		if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX)  in intel_pt_fup_event()
1351 			decoder->state.type |= INTEL_PT_BRANCH;  in intel_pt_fup_event()
1352 		decoder->state.flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1355 	if (decoder->set_fup_ptw) {  in intel_pt_fup_event()
1356 		decoder->set_fup_ptw = false;  in intel_pt_fup_event()
1357 		decoder->state.type |= INTEL_PT_PTW;  in intel_pt_fup_event()
1358 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1359 		decoder->state.ptw_payload = decoder->fup_ptw_payload;  in intel_pt_fup_event()
1362 	if (decoder->set_fup_mwait) {  in intel_pt_fup_event()
1363 		decoder->set_fup_mwait = false;  in intel_pt_fup_event()
1364 		decoder->state.type |= INTEL_PT_MWAIT_OP;  in intel_pt_fup_event()
1365 		decoder->state.mwait_payload = decoder->fup_mwait_payload;  in intel_pt_fup_event()
1368 	if (decoder->set_fup_pwre) {  in intel_pt_fup_event()
1369 		decoder->set_fup_pwre = false;  in intel_pt_fup_event()
1370 		decoder->state.type |= INTEL_PT_PWR_ENTRY;  in intel_pt_fup_event()
1371 		decoder->state.pwre_payload = decoder->fup_pwre_payload;  in intel_pt_fup_event()
1374 	if (decoder->set_fup_exstop) {  in intel_pt_fup_event()
1375 		decoder->set_fup_exstop = false;  in intel_pt_fup_event()
1376 		decoder->state.type |= INTEL_PT_EX_STOP;  in intel_pt_fup_event()
1377 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1380 	if (decoder->set_fup_bep) {  in intel_pt_fup_event()
1381 		decoder->set_fup_bep = false;  in intel_pt_fup_event()
1382 		decoder->state.type |= INTEL_PT_BLK_ITEMS;  in intel_pt_fup_event()
1385 	if (decoder->overflow) {  in intel_pt_fup_event()
1386 		decoder->overflow = false;  in intel_pt_fup_event()
1387 		if (!ret && !decoder->pge) {  in intel_pt_fup_event()
1388 			if (decoder->hop) {  in intel_pt_fup_event()
1389 				decoder->state.type = 0;  in intel_pt_fup_event()
1390 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_fup_event()
1392 			decoder->pge = true;  in intel_pt_fup_event()
1393 			decoder->state.type |= INTEL_PT_BRANCH | INTEL_PT_TRACE_BEGIN;  in intel_pt_fup_event()
1394 			decoder->state.from_ip = 0;  in intel_pt_fup_event()
1395 			decoder->state.to_ip = decoder->ip;  in intel_pt_fup_event()
1400 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1401 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1403 			intel_pt_sample_fup_insn(decoder);  in intel_pt_fup_event()
1405 		decoder->state.type = type;  in intel_pt_fup_event()
1410 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,  in intel_pt_fup_with_nlip()  argument
1414 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&  in intel_pt_fup_with_nlip()
1416 	       ip == decoder->ip + intel_pt_insn->length;  in intel_pt_fup_with_nlip()
1419 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup()  argument
1425 	ip = decoder->last_ip;  in intel_pt_walk_fup()
1428 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);  in intel_pt_walk_fup()
1432 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {  in intel_pt_walk_fup()
1433 			bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP;  in intel_pt_walk_fup()
1435 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_fup()
1436 			if (intel_pt_fup_event(decoder, no_tip) && no_tip)  in intel_pt_walk_fup()
1440 		decoder->set_fup_tx_flags = false;  in intel_pt_walk_fup()
1446 					decoder->ip);  in intel_pt_walk_fup()
1447 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1453 					decoder->ip);  in intel_pt_walk_fup()
1454 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1458 		intel_pt_bug(decoder);  in intel_pt_walk_fup()
1462 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_tip()  argument
1467 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tip()
1469 	    decoder->pgd_ip &&  in intel_pt_walk_tip()
1470 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1471 	    (decoder->state.type & INTEL_PT_BRANCH) &&  in intel_pt_walk_tip()
1472 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {  in intel_pt_walk_tip()
1474 		decoder->no_progress = 0;  in intel_pt_walk_tip()
1475 		decoder->pge = false;  in intel_pt_walk_tip()
1476 		decoder->continuous_period = false;  in intel_pt_walk_tip()
1477 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1478 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1479 		intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1487 	intel_pt_update_nr(decoder);  in intel_pt_walk_tip()
1488 	intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_tip()
1491 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {  in intel_pt_walk_tip()
1492 			decoder->pge = false;  in intel_pt_walk_tip()
1493 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1494 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1495 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1496 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1497 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1499 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1500 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1502 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1504 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1505 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1506 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1507 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1509 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1510 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1517 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +  in intel_pt_walk_tip()
1520 		if (decoder->pgd_ip &&  in intel_pt_walk_tip()
1521 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1522 		    decoder->pgd_ip(to_ip, decoder->data)) {  in intel_pt_walk_tip()
1524 			decoder->pge = false;  in intel_pt_walk_tip()
1525 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1526 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1527 			decoder->ip = to_ip;  in intel_pt_walk_tip()
1528 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1529 			decoder->state.to_ip = to_ip;  in intel_pt_walk_tip()
1530 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1534 				decoder->ip);  in intel_pt_walk_tip()
1535 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_tip()
1539 	return intel_pt_bug(decoder);  in intel_pt_walk_tip()
1605 static int intel_pt_emulated_ptwrite(struct intel_pt_decoder *decoder)  in intel_pt_emulated_ptwrite()  argument
1607 	int n = 64 - decoder->tnt.count;  in intel_pt_emulated_ptwrite()
1610 		.payload = decoder->tnt.payload >> n,  in intel_pt_emulated_ptwrite()
1613 	decoder->emulated_ptwrite = false;  in intel_pt_emulated_ptwrite()
1616 	intel_pt_pkt_lookahead(decoder, intel_pt_eptw_lookahead_cb, &data);  in intel_pt_emulated_ptwrite()
1620 	decoder->state.type = INTEL_PT_PTW;  in intel_pt_emulated_ptwrite()
1621 	decoder->state.from_ip = decoder->ip;  in intel_pt_emulated_ptwrite()
1622 	decoder->state.to_ip = 0;  in intel_pt_emulated_ptwrite()
1623 	decoder->state.ptw_payload = data.payload;  in intel_pt_emulated_ptwrite()
1627 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)  in intel_pt_walk_tnt()  argument
1633 		if (decoder->emulated_ptwrite)  in intel_pt_walk_tnt()
1634 			return intel_pt_emulated_ptwrite(decoder);  in intel_pt_walk_tnt()
1635 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tnt()
1637 			decoder->emulated_ptwrite = intel_pt_insn.emulated_ptwrite;  in intel_pt_walk_tnt()
1641 			decoder->emulated_ptwrite = false;  in intel_pt_walk_tnt()
1646 			if (!decoder->return_compression) {  in intel_pt_walk_tnt()
1648 						decoder->ip);  in intel_pt_walk_tnt()
1649 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1652 			if (!decoder->ret_addr) {  in intel_pt_walk_tnt()
1654 						decoder->ip);  in intel_pt_walk_tnt()
1655 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1658 			if (!(decoder->tnt.payload & BIT63)) {  in intel_pt_walk_tnt()
1660 						decoder->ip);  in intel_pt_walk_tnt()
1661 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1664 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1665 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1666 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1668 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1669 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1670 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1671 			decoder->ip = decoder->ret_addr;  in intel_pt_walk_tnt()
1672 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1678 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_tnt()
1681 			if (decoder->packet.type != INTEL_PT_TIP ||  in intel_pt_walk_tnt()
1682 			    decoder->packet.count == 0) {  in intel_pt_walk_tnt()
1684 						decoder->ip);  in intel_pt_walk_tnt()
1685 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1686 				decoder->pkt_step = 0;  in intel_pt_walk_tnt()
1689 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_tnt()
1690 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1691 			decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tnt()
1692 			decoder->ip = decoder->last_ip;  in intel_pt_walk_tnt()
1693 			intel_pt_update_nr(decoder);  in intel_pt_walk_tnt()
1694 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_tnt()
1699 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1700 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1701 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1703 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1704 			if (decoder->tnt.payload & BIT63) {  in intel_pt_walk_tnt()
1705 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1706 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1707 				decoder->ip += intel_pt_insn.length +  in intel_pt_walk_tnt()
1709 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1713 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {  in intel_pt_walk_tnt()
1714 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1715 				decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_tnt()
1716 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1717 				decoder->state.to_ip = 0;  in intel_pt_walk_tnt()
1718 				decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1721 			decoder->sample_cyc = false;  in intel_pt_walk_tnt()
1722 			decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1723 			if (!decoder->tnt.count) {  in intel_pt_walk_tnt()
1724 				intel_pt_update_sample_time(decoder);  in intel_pt_walk_tnt()
1727 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1731 		return intel_pt_bug(decoder);  in intel_pt_walk_tnt()
1735 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)  in intel_pt_mode_tsx()  argument
1740 	fup_tx_flags = decoder->packet.payload &  in intel_pt_mode_tsx()
1742 	err = intel_pt_get_next_packet(decoder);  in intel_pt_mode_tsx()
1745 	if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_mode_tsx()
1746 		decoder->fup_tx_flags = fup_tx_flags;  in intel_pt_mode_tsx()
1747 		decoder->set_fup_tx_flags = true;  in intel_pt_mode_tsx()
1748 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))  in intel_pt_mode_tsx()
1752 				decoder->pos);  in intel_pt_mode_tsx()
1753 		intel_pt_update_in_tx(decoder);  in intel_pt_mode_tsx()
1758 static int intel_pt_evd(struct intel_pt_decoder *decoder)  in intel_pt_evd()  argument
1760 	if (decoder->evd_cnt >= INTEL_PT_MAX_EVDS) {  in intel_pt_evd()
1761 		intel_pt_log_at("ERROR: Too many EVD packets", decoder->pos);  in intel_pt_evd()
1764 	decoder->evd[decoder->evd_cnt++] = (struct intel_pt_evd){  in intel_pt_evd()
1765 		.type = decoder->packet.count,  in intel_pt_evd()
1766 		.payload = decoder->packet.payload,  in intel_pt_evd()
1786 /* For use only when decoder->vm_time_correlation is true */
1787 static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder,  in intel_pt_time_in_range()  argument
1790 	uint64_t max_timestamp = decoder->buf_timestamp;  in intel_pt_time_in_range()
1793 		max_timestamp = decoder->last_reliable_timestamp +  in intel_pt_time_in_range()
1796 	return timestamp >= decoder->last_reliable_timestamp &&  in intel_pt_time_in_range()
1797 	       timestamp < decoder->buf_timestamp;  in intel_pt_time_in_range()
1800 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_tsc_timestamp()  argument
1805 	decoder->have_tma = false;  in intel_pt_calc_tsc_timestamp()
1807 	if (decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1808 		timestamp = intel_pt_8b_tsc(decoder->packet.payload,  in intel_pt_calc_tsc_timestamp()
1809 					    decoder->ref_timestamp);  in intel_pt_calc_tsc_timestamp()
1810 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1811 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1812 		decoder->ref_timestamp = 0;  in intel_pt_calc_tsc_timestamp()
1813 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1814 	} else if (decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1815 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1816 			    (decoder->timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1817 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1818 		if (timestamp < decoder->timestamp &&  in intel_pt_calc_tsc_timestamp()
1819 		    decoder->timestamp - timestamp < decoder->tsc_slip) {  in intel_pt_calc_tsc_timestamp()
1822 			timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1824 		if (timestamp < decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1825 			if (!decoder->buf_timestamp ||  in intel_pt_calc_tsc_timestamp()
1826 			    (timestamp + (1ULL << 56) < decoder->buf_timestamp)) {  in intel_pt_calc_tsc_timestamp()
1829 				decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1832 				timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1836 		if (decoder->vm_time_correlation &&  in intel_pt_calc_tsc_timestamp()
1837 		    (bad || !intel_pt_time_in_range(decoder, timestamp)) &&  in intel_pt_calc_tsc_timestamp()
1838 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_calc_tsc_timestamp()
1840 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1841 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1844 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_tsc_timestamp()
1845 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1846 		decoder->cycle_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1847 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_tsc_timestamp()
1848 		intel_pt_calc_cyc_to_tsc(decoder, false);  in intel_pt_calc_tsc_timestamp()
1851 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_tsc_timestamp()
1854 static int intel_pt_overflow(struct intel_pt_decoder *decoder)  in intel_pt_overflow()  argument
1857 	intel_pt_clear_tx_flags(decoder);  in intel_pt_overflow()
1858 	intel_pt_set_nr(decoder);  in intel_pt_overflow()
1859 	decoder->timestamp_insn_cnt = 0;  in intel_pt_overflow()
1860 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_overflow()
1861 	decoder->state.from_ip = decoder->ip;  in intel_pt_overflow()
1862 	decoder->ip = 0;  in intel_pt_overflow()
1863 	decoder->pge = false;  in intel_pt_overflow()
1864 	intel_pt_clear_fup_event(decoder);  in intel_pt_overflow()
1865 	decoder->overflow = true;  in intel_pt_overflow()
1869 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_pge()  argument
1871 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_pge()
1874 	decoder->cyc_cnt_timestamp = decoder->timestamp;  in intel_pt_mtc_cyc_cnt_pge()
1875 	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_pge()
1878 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_cbr()  argument
1880 	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;  in intel_pt_mtc_cyc_cnt_cbr()
1882 	if (decoder->pge)  in intel_pt_mtc_cyc_cnt_cbr()
1883 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_mtc_cyc_cnt_cbr()
1886 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_upd()  argument
1890 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_upd()
1893 	decoder->sample_cyc = true;  in intel_pt_mtc_cyc_cnt_upd()
1895 	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)  in intel_pt_mtc_cyc_cnt_upd()
1898 	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;  in intel_pt_mtc_cyc_cnt_upd()
1899 	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1901 	if (tot_cyc_cnt > decoder->tot_cyc_cnt)  in intel_pt_mtc_cyc_cnt_upd()
1902 		decoder->tot_cyc_cnt = tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1905 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)  in intel_pt_calc_tma()  argument
1907 	uint32_t ctc = decoder->packet.payload;  in intel_pt_calc_tma()
1908 	uint32_t fc = decoder->packet.count;  in intel_pt_calc_tma()
1909 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_tma()
1911 	if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_tma()
1914 	if (decoder->pge && !decoder->in_psb)  in intel_pt_calc_tma()
1915 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_calc_tma()
1917 		intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_tma()
1919 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_tma()
1920 	decoder->last_ctc = ctc - ctc_rem;  in intel_pt_calc_tma()
1921 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;  in intel_pt_calc_tma()
1922 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_tma()
1923 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_tma()
1925 		decoder->ctc_timestamp -= multdiv(ctc_rem,  in intel_pt_calc_tma()
1926 						  decoder->tsc_ctc_ratio_n,  in intel_pt_calc_tma()
1927 						  decoder->tsc_ctc_ratio_d);  in intel_pt_calc_tma()
1929 	decoder->ctc_delta = 0;  in intel_pt_calc_tma()
1930 	decoder->have_tma = true;  in intel_pt_calc_tma()
1931 	decoder->fixup_last_mtc = true;  in intel_pt_calc_tma()
1933 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);  in intel_pt_calc_tma()
1936 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_mtc_timestamp()  argument
1941 	if (!decoder->have_tma)  in intel_pt_calc_mtc_timestamp()
1944 	mtc = decoder->packet.payload;  in intel_pt_calc_mtc_timestamp()
1946 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {  in intel_pt_calc_mtc_timestamp()
1947 		decoder->fixup_last_mtc = false;  in intel_pt_calc_mtc_timestamp()
1948 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_mtc_timestamp()
1949 					&decoder->last_mtc);  in intel_pt_calc_mtc_timestamp()
1952 	if (mtc > decoder->last_mtc)  in intel_pt_calc_mtc_timestamp()
1953 		mtc_delta = mtc - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1955 		mtc_delta = mtc + 256 - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1957 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_mtc_timestamp()
1959 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_mtc_timestamp()
1960 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1961 			    decoder->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_mtc_timestamp()
1963 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1964 			    multdiv(decoder->ctc_delta,  in intel_pt_calc_mtc_timestamp()
1965 				    decoder->tsc_ctc_ratio_n,  in intel_pt_calc_mtc_timestamp()
1966 				    decoder->tsc_ctc_ratio_d);  in intel_pt_calc_mtc_timestamp()
1969 	if (timestamp < decoder->timestamp)  in intel_pt_calc_mtc_timestamp()
1971 			     timestamp, decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1973 		decoder->timestamp = timestamp;  in intel_pt_calc_mtc_timestamp()
1975 	intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_mtc_timestamp()
1977 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1978 	decoder->last_mtc = mtc;  in intel_pt_calc_mtc_timestamp()
1980 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_mtc_timestamp()
1981 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_mtc_timestamp()
1982 		decoder->cycle_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1983 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_mtc_timestamp()
1984 		intel_pt_calc_cyc_to_tsc(decoder, true);  in intel_pt_calc_mtc_timestamp()
1987 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1990 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)  in intel_pt_calc_cbr()  argument
1992 	unsigned int cbr = decoder->packet.payload & 0xff;  in intel_pt_calc_cbr()
1994 	decoder->cbr_payload = decoder->packet.payload;  in intel_pt_calc_cbr()
1996 	if (decoder->cbr == cbr)  in intel_pt_calc_cbr()
1999 	decoder->cbr = cbr;  in intel_pt_calc_cbr()
2000 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cbr()
2001 	decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_cbr()
2002 	decoder->cycle_cnt = 0;  in intel_pt_calc_cbr()
2004 	intel_pt_mtc_cyc_cnt_cbr(decoder);  in intel_pt_calc_cbr()
2007 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_cyc_timestamp()  argument
2009 	uint64_t timestamp = decoder->cyc_ref_timestamp;  in intel_pt_calc_cyc_timestamp()
2011 	decoder->have_cyc = true;  in intel_pt_calc_cyc_timestamp()
2013 	decoder->cycle_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
2014 	if (decoder->pge)  in intel_pt_calc_cyc_timestamp()
2015 		decoder->tot_cyc_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
2016 	decoder->sample_cyc = true;  in intel_pt_calc_cyc_timestamp()
2018 	if (!decoder->cyc_ref_timestamp)  in intel_pt_calc_cyc_timestamp()
2021 	if (decoder->have_calc_cyc_to_tsc)  in intel_pt_calc_cyc_timestamp()
2022 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
2023 	else if (decoder->cbr)  in intel_pt_calc_cyc_timestamp()
2024 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
2028 	if (timestamp < decoder->timestamp)  in intel_pt_calc_cyc_timestamp()
2030 			     timestamp, decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
2032 		decoder->timestamp = timestamp;  in intel_pt_calc_cyc_timestamp()
2034 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_cyc_timestamp()
2036 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
2039 static void intel_pt_bbp(struct intel_pt_decoder *decoder)  in intel_pt_bbp()  argument
2041 	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {  in intel_pt_bbp()
2042 		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));  in intel_pt_bbp()
2043 		decoder->state.items.is_32_bit = false;  in intel_pt_bbp()
2045 	decoder->blk_type = decoder->packet.payload;  in intel_pt_bbp()
2046 	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);  in intel_pt_bbp()
2047 	if (decoder->blk_type == INTEL_PT_GP_REGS)  in intel_pt_bbp()
2048 		decoder->state.items.is_32_bit = decoder->packet.count;  in intel_pt_bbp()
2049 	if (decoder->blk_type_pos < 0) {  in intel_pt_bbp()
2051 			     decoder->blk_type);  in intel_pt_bbp()
2052 	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {  in intel_pt_bbp()
2054 			     decoder->blk_type);  in intel_pt_bbp()
2058 static void intel_pt_bip(struct intel_pt_decoder *decoder)  in intel_pt_bip()  argument
2060 	uint32_t id = decoder->packet.count;  in intel_pt_bip()
2062 	int pos = decoder->blk_type_pos;  in intel_pt_bip()
2066 			     id, decoder->blk_type);  in intel_pt_bip()
2070 	if (decoder->state.items.mask[pos] & bit) {  in intel_pt_bip()
2072 			     id, decoder->blk_type);  in intel_pt_bip()
2075 	decoder->state.items.mask[pos] |= bit;  in intel_pt_bip()
2076 	decoder->state.items.val[pos][id] = decoder->packet.payload;  in intel_pt_bip()
2080 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)  in intel_pt_walk_psbend()  argument
2084 	decoder->in_psb = true;  in intel_pt_walk_psbend()
2087 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psbend()
2091 		switch (decoder->packet.type) {  in intel_pt_walk_psbend()
2117 			decoder->have_tma = false;  in intel_pt_walk_psbend()
2123 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psbend()
2127 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psbend()
2131 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psbend()
2135 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psbend()
2139 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_psbend()
2143 			intel_pt_set_pip(decoder);  in intel_pt_walk_psbend()
2147 			decoder->pge = true;  in intel_pt_walk_psbend()
2148 			if (decoder->packet.count) {  in intel_pt_walk_psbend()
2149 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_psbend()
2150 				decoder->psb_ip = decoder->last_ip;  in intel_pt_walk_psbend()
2155 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psbend()
2159 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psbend()
2160 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_psbend()
2161 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_psbend()
2165 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psbend()
2176 	decoder->in_psb = false;  in intel_pt_walk_psbend()
2181 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup_tip()  argument
2185 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {  in intel_pt_walk_fup_tip()
2186 		decoder->tx_flags = 0;  in intel_pt_walk_fup_tip()
2187 		decoder->state.flags &= ~INTEL_PT_IN_TX;  in intel_pt_walk_fup_tip()
2188 		decoder->state.flags |= INTEL_PT_ABORT_TX;  in intel_pt_walk_fup_tip()
2190 		decoder->state.flags |= INTEL_PT_ASYNC;  in intel_pt_walk_fup_tip()
2194 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_fup_tip()
2198 		switch (decoder->packet.type) {  in intel_pt_walk_fup_tip()
2223 			decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_fup_tip()
2224 			decoder->pkt_step = 0;  in intel_pt_walk_fup_tip()
2228 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_fup_tip()
2232 			return intel_pt_overflow(decoder);  in intel_pt_walk_fup_tip()
2235 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2236 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2237 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2239 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2240 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2242 			decoder->pge = false;  in intel_pt_walk_fup_tip()
2243 			decoder->continuous_period = false;  in intel_pt_walk_fup_tip()
2244 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_fup_tip()
2245 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
2249 			decoder->pge = true;  in intel_pt_walk_fup_tip()
2251 				     decoder->ip);  in intel_pt_walk_fup_tip()
2252 			decoder->state.from_ip = 0;  in intel_pt_walk_fup_tip()
2253 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2254 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2256 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2257 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2259 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_fup_tip()
2260 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_fup_tip()
2261 			intel_pt_set_nr(decoder);  in intel_pt_walk_fup_tip()
2265 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2266 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
2267 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
2269 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
2270 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
2272 			intel_pt_update_nr(decoder);  in intel_pt_walk_fup_tip()
2273 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_fup_tip()
2277 			intel_pt_update_pip(decoder);  in intel_pt_walk_fup_tip()
2281 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2282 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_fup_tip()
2283 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_fup_tip()
2287 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_fup_tip()
2291 			intel_pt_mode_exec(decoder);  in intel_pt_walk_fup_tip()
2300 			return intel_pt_bug(decoder);  in intel_pt_walk_fup_tip()
2305 static int intel_pt_resample(struct intel_pt_decoder *decoder)  in intel_pt_resample()  argument
2307 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_resample()
2308 	decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_resample()
2309 	decoder->state.from_ip = decoder->ip;  in intel_pt_resample()
2310 	decoder->state.to_ip = 0;  in intel_pt_resample()
2410 static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder)  in intel_pt_ovf_fup_lookahead()  argument
2417 	intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data);  in intel_pt_ovf_fup_lookahead()
2437 static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc)  in intel_pt_ctc_to_tsc()  argument
2439 	if (decoder->tsc_ctc_mult)  in intel_pt_ctc_to_tsc()
2440 		return ctc * decoder->tsc_ctc_mult;  in intel_pt_ctc_to_tsc()
2442 		return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d);  in intel_pt_ctc_to_tsc()
2445 static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_expected_tsc()  argument
2468 	expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc;  in intel_pt_calc_expected_tsc()
2479 		intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta));  in intel_pt_calc_expected_tsc()
2486 static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder,  in intel_pt_expected_tsc()  argument
2492 	return intel_pt_calc_expected_tsc(decoder, ctc, fc,  in intel_pt_expected_tsc()
2493 					  decoder->ctc_timestamp,  in intel_pt_expected_tsc()
2497 static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc()  argument
2500 	uint64_t payload = decoder->packet.payload;  in intel_pt_translate_vm_tsc()
2503 	decoder->packet.payload -= vmcs_info->tsc_offset;  in intel_pt_translate_vm_tsc()
2505 	decoder->packet.payload &= SEVEN_BYTES;  in intel_pt_translate_vm_tsc()
2511 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_translate_vm_tsc()
2512 		memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7);  in intel_pt_translate_vm_tsc()
2516 		     payload, decoder->packet.payload, vmcs_info->vmcs,  in intel_pt_translate_vm_tsc()
2520 static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder,  in intel_pt_translate_vm_tsc_offset()  argument
2528 	intel_pt_translate_vm_tsc(decoder, &vmcs_info);  in intel_pt_translate_vm_tsc_offset()
2547 static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_psb()  argument
2551 	data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_psb()
2552 	data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_psb()
2553 	intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data);  in intel_pt_vm_tm_corr_psb()
2556 	decoder->in_psb = data->psbend;  in intel_pt_vm_tm_corr_psb()
2559 static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_first_tsc()  argument
2564 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_first_tsc()
2572 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2573 				decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2578 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_first_tsc()
2581 		decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_first_tsc()
2582 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_first_tsc()
2587 				intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_first_tsc()
2594 static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder,  in intel_pt_vm_tm_corr_tsc()  argument
2609 	if (!decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2611 		data->ctc_delta = decoder->ctc_delta;  in intel_pt_vm_tm_corr_tsc()
2612 		data->last_ctc = decoder->last_ctc;  in intel_pt_vm_tm_corr_tsc()
2614 		intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data);  in intel_pt_vm_tm_corr_tsc()
2615 		if (decoder->pge) {  in intel_pt_vm_tm_corr_tsc()
2617 			data->pip_packet.payload = decoder->pip_payload;  in intel_pt_vm_tm_corr_tsc()
2627 	vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs;  in intel_pt_vm_tm_corr_tsc()
2631 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_tsc()
2633 	ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp;  in intel_pt_vm_tm_corr_tsc()
2634 	host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp);  in intel_pt_vm_tm_corr_tsc()
2636 	if (!decoder->ctc_timestamp) {  in intel_pt_vm_tm_corr_tsc()
2637 		intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc);  in intel_pt_vm_tm_corr_tsc()
2641 	expected_tsc = intel_pt_expected_tsc(decoder, data);  in intel_pt_vm_tm_corr_tsc()
2651 				intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2652 				decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2656 			decoder->last_reliable_timestamp = host_tsc;  in intel_pt_vm_tm_corr_tsc()
2657 			decoder->vm_tm_corr_reliable = true;  in intel_pt_vm_tm_corr_tsc()
2662 		if (decoder->in_psb) {  in intel_pt_vm_tm_corr_tsc()
2685 		if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_tsc()
2687 			decoder->vm_tm_corr_reliable = false;  in intel_pt_vm_tm_corr_tsc()
2698 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable &&  in intel_pt_vm_tm_corr_tsc()
2699 			   decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2703 		} else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) {  in intel_pt_vm_tm_corr_tsc()
2728 		intel_pt_translate_vm_tsc(decoder, vmcs_info);  in intel_pt_vm_tm_corr_tsc()
2738 			if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_tsc()
2741 		intel_pt_translate_vm_tsc_offset(decoder, tsc_offset);  in intel_pt_vm_tm_corr_tsc()
2744 	decoder->vm_tm_corr_reliable = reliable;  in intel_pt_vm_tm_corr_tsc()
2747 static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder)  in intel_pt_vm_tm_corr_pebs_tsc()  argument
2749 	uint64_t host_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2750 	uint64_t guest_tsc = decoder->packet.payload;  in intel_pt_vm_tm_corr_pebs_tsc()
2754 	vmcs = decoder->vmcs;  in intel_pt_vm_tm_corr_pebs_tsc()
2758 	vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs);  in intel_pt_vm_tm_corr_pebs_tsc()
2760 	if (decoder->pge) {  in intel_pt_vm_tm_corr_pebs_tsc()
2761 		if (in_vm(decoder->pip_payload)) { /* Guest */  in intel_pt_vm_tm_corr_pebs_tsc()
2770 		if (intel_pt_time_in_range(decoder, host_tsc)) {  in intel_pt_vm_tm_corr_pebs_tsc()
2779 		host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp);  in intel_pt_vm_tm_corr_pebs_tsc()
2784 		if (!intel_pt_time_in_range(decoder, host_tsc) &&  in intel_pt_vm_tm_corr_pebs_tsc()
2785 		    intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE))  in intel_pt_vm_tm_corr_pebs_tsc()
2788 		if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS))  in intel_pt_vm_tm_corr_pebs_tsc()
2790 		host_tsc = decoder->timestamp;  in intel_pt_vm_tm_corr_pebs_tsc()
2793 	decoder->packet.payload = host_tsc;  in intel_pt_vm_tm_corr_pebs_tsc()
2795 	if (!decoder->vm_tm_corr_dry_run)  in intel_pt_vm_tm_corr_pebs_tsc()
2796 		memcpy((void *)decoder->buf + 1, &host_tsc, 8);  in intel_pt_vm_tm_corr_pebs_tsc()
2799 static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder)  in intel_pt_vm_time_correlation()  argument
2805 	if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2806 		intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2809 		err = intel_pt_get_next_packet(decoder);  in intel_pt_vm_time_correlation()
2815 		switch (decoder->packet.type) {  in intel_pt_vm_time_correlation()
2817 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2818 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2824 			decoder->pge = true;  in intel_pt_vm_time_correlation()
2828 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2829 			pge = decoder->pge;  in intel_pt_vm_time_correlation()
2830 			decoder->pge = intel_pt_ovf_fup_lookahead(decoder);  in intel_pt_vm_time_correlation()
2831 			if (pge != decoder->pge)  in intel_pt_vm_time_correlation()
2833 			if (!decoder->pge)  in intel_pt_vm_time_correlation()
2834 				decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2838 			if (decoder->in_psb)  in intel_pt_vm_time_correlation()
2839 				decoder->pge = true;  in intel_pt_vm_time_correlation()
2843 			decoder->pge = false;  in intel_pt_vm_time_correlation()
2844 			decoder->vm_tm_corr_continuous = false;  in intel_pt_vm_time_correlation()
2845 			decoder->have_tma = false;  in intel_pt_vm_time_correlation()
2849 			intel_pt_vm_tm_corr_psb(decoder, &data);  in intel_pt_vm_time_correlation()
2853 			decoder->pip_payload = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2857 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2861 			intel_pt_vm_tm_corr_tsc(decoder, &data);  in intel_pt_vm_time_correlation()
2862 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2863 			decoder->vm_tm_corr_same_buf = true;  in intel_pt_vm_time_correlation()
2864 			decoder->vm_tm_corr_continuous = decoder->pge;  in intel_pt_vm_time_correlation()
2868 			intel_pt_calc_tma(decoder);  in intel_pt_vm_time_correlation()
2872 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_vm_time_correlation()
2876 			intel_pt_calc_cbr(decoder);  in intel_pt_vm_time_correlation()
2880 			decoder->in_psb = false;  in intel_pt_vm_time_correlation()
2885 			if (decoder->packet.payload != NO_VMCS)  in intel_pt_vm_time_correlation()
2886 				decoder->vmcs = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2890 			decoder->blk_type = decoder->packet.payload;  in intel_pt_vm_time_correlation()
2894 			if (decoder->blk_type == INTEL_PT_PEBS_BASIC &&  in intel_pt_vm_time_correlation()
2895 			    decoder->packet.count == 2)  in intel_pt_vm_time_correlation()
2896 				intel_pt_vm_tm_corr_pebs_tsc(decoder);  in intel_pt_vm_time_correlation()
2901 			decoder->blk_type = 0;  in intel_pt_vm_time_correlation()
2932 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
2935 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)  in intel_pt_hop_trace()  argument
2940 	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {  in intel_pt_hop_trace()
2941 		*err = intel_pt_scan_for_psb(decoder);  in intel_pt_hop_trace()
2946 	switch (decoder->packet.type) {  in intel_pt_hop_trace()
2951 		decoder->pge = false;  in intel_pt_hop_trace()
2952 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2953 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2956 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2957 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_hop_trace()
2958 		decoder->state.from_ip = 0;  in intel_pt_hop_trace()
2959 		decoder->state.to_ip = decoder->ip;  in intel_pt_hop_trace()
2960 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2964 		if (!decoder->packet.count) {  in intel_pt_hop_trace()
2965 			intel_pt_set_nr(decoder);  in intel_pt_hop_trace()
2968 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2969 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2970 		decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2971 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2972 		intel_pt_update_nr(decoder);  in intel_pt_hop_trace()
2973 		intel_pt_sample_iflag_chg(decoder);  in intel_pt_hop_trace()
2977 		if (!decoder->packet.count)  in intel_pt_hop_trace()
2979 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
2980 		if (decoder->set_fup_mwait || decoder->set_fup_pwre)  in intel_pt_hop_trace()
2982 		if (!decoder->branch_enable || !decoder->pge)  in intel_pt_hop_trace()
2985 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
2986 			decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
2987 			decoder->state.to_ip = 0;  in intel_pt_hop_trace()
2988 			intel_pt_fup_event(decoder, *no_tip);  in intel_pt_hop_trace()
2991 		intel_pt_fup_event(decoder, *no_tip);  in intel_pt_hop_trace()
2992 		decoder->state.type |= INTEL_PT_INSTRUCTION | INTEL_PT_BRANCH;  in intel_pt_hop_trace()
2993 		*err = intel_pt_walk_fup_tip(decoder);  in intel_pt_hop_trace()
2994 		if (!*err && decoder->state.to_ip)  in intel_pt_hop_trace()
2995 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_hop_trace()
2999 		decoder->state.psb_offset = decoder->pos;  in intel_pt_hop_trace()
3000 		decoder->psb_ip = 0;  in intel_pt_hop_trace()
3001 		decoder->last_ip = 0;  in intel_pt_hop_trace()
3002 		decoder->have_last_ip = true;  in intel_pt_hop_trace()
3003 		*err = intel_pt_walk_psbend(decoder);  in intel_pt_hop_trace()
3008 		decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_hop_trace()
3009 		if (decoder->psb_ip) {  in intel_pt_hop_trace()
3010 			decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
3011 			decoder->ip = decoder->psb_ip;  in intel_pt_hop_trace()
3013 		decoder->state.from_ip = decoder->psb_ip;  in intel_pt_hop_trace()
3014 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
3140 static int intel_pt_psb(struct intel_pt_decoder *decoder)  in intel_pt_psb()  argument
3144 	decoder->last_ip = 0;  in intel_pt_psb()
3145 	decoder->psb_ip = 0;  in intel_pt_psb()
3146 	decoder->have_last_ip = true;  in intel_pt_psb()
3147 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_psb()
3148 	err = intel_pt_walk_psbend(decoder);  in intel_pt_psb()
3151 	decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_psb()
3152 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_psb()
3153 	decoder->state.to_ip = 0;  in intel_pt_psb()
3157 static int intel_pt_fup_in_psb(struct intel_pt_decoder *decoder)  in intel_pt_fup_in_psb()  argument
3161 	if (decoder->ip != decoder->last_ip) {  in intel_pt_fup_in_psb()
3162 		err = intel_pt_walk_fup(decoder);  in intel_pt_fup_in_psb()
3167 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_fup_in_psb()
3168 	err = intel_pt_psb(decoder);  in intel_pt_fup_in_psb()
3170 		decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_fup_in_psb()
3177 static bool intel_pt_psb_with_fup(struct intel_pt_decoder *decoder, int *err)  in intel_pt_psb_with_fup()  argument
3181 	if (!decoder->branch_enable)  in intel_pt_psb_with_fup()
3184 	intel_pt_pkt_lookahead(decoder, intel_pt_psb_lookahead_cb, &data);  in intel_pt_psb_with_fup()
3188 	decoder->packet = data.fup_packet;  in intel_pt_psb_with_fup()
3189 	intel_pt_set_last_ip(decoder);  in intel_pt_psb_with_fup()
3190 	decoder->pkt_state = INTEL_PT_STATE_FUP_IN_PSB;  in intel_pt_psb_with_fup()
3192 	*err = intel_pt_fup_in_psb(decoder);  in intel_pt_psb_with_fup()
3197 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)  in intel_pt_walk_trace()  argument
3204 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3209 		if (decoder->cyc_threshold) {  in intel_pt_walk_trace()
3210 			if (decoder->sample_cyc && last_packet_type != INTEL_PT_CYC)  in intel_pt_walk_trace()
3211 				decoder->sample_cyc = false;  in intel_pt_walk_trace()
3212 			last_packet_type = decoder->packet.type;  in intel_pt_walk_trace()
3215 		if (decoder->hop) {  in intel_pt_walk_trace()
3216 			switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {  in intel_pt_walk_trace()
3228 		switch (decoder->packet.type) {  in intel_pt_walk_trace()
3230 			if (!decoder->packet.count)  in intel_pt_walk_trace()
3232 			decoder->tnt = decoder->packet;  in intel_pt_walk_trace()
3233 			decoder->pkt_state = INTEL_PT_STATE_TNT;  in intel_pt_walk_trace()
3234 			err = intel_pt_walk_tnt(decoder);  in intel_pt_walk_trace()
3240 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
3241 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3242 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;  in intel_pt_walk_trace()
3243 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
3246 			decoder->pge = true;  in intel_pt_walk_trace()
3247 			decoder->overflow = false;  in intel_pt_walk_trace()
3248 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_trace()
3249 			intel_pt_set_nr(decoder);  in intel_pt_walk_trace()
3250 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
3252 						decoder->pos);  in intel_pt_walk_trace()
3255 			intel_pt_sample_iflag_chg(decoder);  in intel_pt_walk_trace()
3256 			intel_pt_set_ip(decoder);  in intel_pt_walk_trace()
3257 			decoder->state.from_ip = 0;  in intel_pt_walk_trace()
3258 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_trace()
3259 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_trace()
3264 			if (decoder->hop)  in intel_pt_walk_trace()
3265 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_walk_trace()
3270 			return intel_pt_overflow(decoder);  in intel_pt_walk_trace()
3273 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
3274 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3275 			decoder->pkt_state = INTEL_PT_STATE_TIP;  in intel_pt_walk_trace()
3276 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
3279 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
3281 						decoder->pos);  in intel_pt_walk_trace()
3285 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
3286 			if (!decoder->branch_enable || !decoder->pge) {  in intel_pt_walk_trace()
3287 				decoder->ip = decoder->last_ip;  in intel_pt_walk_trace()
3288 				if (intel_pt_fup_event(decoder, no_tip))  in intel_pt_walk_trace()
3293 			if (decoder->set_fup_mwait)  in intel_pt_walk_trace()
3296 				decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;  in intel_pt_walk_trace()
3298 				decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_walk_trace()
3299 			err = intel_pt_walk_fup(decoder);  in intel_pt_walk_trace()
3306 			return intel_pt_walk_fup_tip(decoder);  in intel_pt_walk_trace()
3309 			decoder->pge = false;  in intel_pt_walk_trace()
3310 			decoder->continuous_period = false;  in intel_pt_walk_trace()
3311 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_trace()
3312 			decoder->have_tma = false;  in intel_pt_walk_trace()
3316 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_trace()
3317 			decoder->psb_ip = 0;  in intel_pt_walk_trace()
3318 			if (intel_pt_psb_with_fup(decoder, &err))  in intel_pt_walk_trace()
3320 			err = intel_pt_psb(decoder);  in intel_pt_walk_trace()
3326 			intel_pt_update_pip(decoder);  in intel_pt_walk_trace()
3330 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_trace()
3331 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)  in intel_pt_walk_trace()
3337 			if (!decoder->mtc_insn)  in intel_pt_walk_trace()
3339 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3341 			if (!decoder->timestamp)  in intel_pt_walk_trace()
3343 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_trace()
3344 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3345 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3346 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
3350 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_trace()
3354 			intel_pt_calc_tma(decoder);  in intel_pt_walk_trace()
3358 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_trace()
3362 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_trace()
3363 			if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
3364 				decoder->state.type = 0;  in intel_pt_walk_trace()
3370 			intel_pt_mode_exec(decoder);  in intel_pt_walk_trace()
3371 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3374 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3375 				decoder->set_fup_mode_exec = true;  in intel_pt_walk_trace()
3382 			if (!decoder->pge || decoder->in_psb) {  in intel_pt_walk_trace()
3383 				intel_pt_update_in_tx(decoder);  in intel_pt_walk_trace()
3386 			err = intel_pt_mode_tsx(decoder, &no_tip);  in intel_pt_walk_trace()
3392 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3401 			decoder->fup_ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3402 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3405 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3406 				decoder->set_fup_ptw = true;  in intel_pt_walk_trace()
3410 						decoder->pos);  in intel_pt_walk_trace()
3415 			decoder->state.type = INTEL_PT_PTW;  in intel_pt_walk_trace()
3416 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3417 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3418 			decoder->state.ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3422 			decoder->fup_mwait_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3423 			decoder->set_fup_mwait = true;  in intel_pt_walk_trace()
3427 			if (decoder->set_fup_mwait) {  in intel_pt_walk_trace()
3428 				decoder->fup_pwre_payload =  in intel_pt_walk_trace()
3429 							decoder->packet.payload;  in intel_pt_walk_trace()
3430 				decoder->set_fup_pwre = true;  in intel_pt_walk_trace()
3433 			decoder->state.type = INTEL_PT_PWR_ENTRY;  in intel_pt_walk_trace()
3434 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3435 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3436 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3440 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3443 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3444 				decoder->set_fup_exstop = true;  in intel_pt_walk_trace()
3448 						decoder->pos);  in intel_pt_walk_trace()
3453 			decoder->state.type = INTEL_PT_EX_STOP;  in intel_pt_walk_trace()
3454 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3455 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3459 			decoder->state.type = INTEL_PT_PWR_EXIT;  in intel_pt_walk_trace()
3460 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3461 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3462 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
3466 			intel_pt_bbp(decoder);  in intel_pt_walk_trace()
3470 			intel_pt_bip(decoder);  in intel_pt_walk_trace()
3474 			decoder->state.type = INTEL_PT_BLK_ITEMS;  in intel_pt_walk_trace()
3475 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
3476 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
3480 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3483 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3484 				decoder->set_fup_bep = true;  in intel_pt_walk_trace()
3488 						decoder->pos);  in intel_pt_walk_trace()
3493 			decoder->fup_cfe_pkt = decoder->packet;  in intel_pt_walk_trace()
3494 			decoder->set_fup_cfe = true;  in intel_pt_walk_trace()
3495 			if (!decoder->pge) {  in intel_pt_walk_trace()
3496 				intel_pt_fup_event(decoder, true);  in intel_pt_walk_trace()
3502 			decoder->fup_cfe_pkt = decoder->packet;  in intel_pt_walk_trace()
3503 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
3506 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
3507 				decoder->set_fup_cfe_ip = true;  in intel_pt_walk_trace()
3511 						decoder->pos);  in intel_pt_walk_trace()
3516 			err = intel_pt_evd(decoder);  in intel_pt_walk_trace()
3522 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
3527 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)  in intel_pt_have_ip()  argument
3529 	return decoder->packet.count &&  in intel_pt_have_ip()
3530 	       (decoder->have_last_ip || decoder->packet.count == 3 ||  in intel_pt_have_ip()
3531 		decoder->packet.count == 6);  in intel_pt_have_ip()
3535 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)  in intel_pt_walk_psb()  argument
3539 	decoder->in_psb = true;  in intel_pt_walk_psb()
3542 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psb()
3546 		switch (decoder->packet.type) {  in intel_pt_walk_psb()
3548 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3571 			decoder->pge = true;  in intel_pt_walk_psb()
3572 			if (intel_pt_have_ip(decoder)) {  in intel_pt_walk_psb()
3573 				uint64_t current_ip = decoder->ip;  in intel_pt_walk_psb()
3575 				intel_pt_set_ip(decoder);  in intel_pt_walk_psb()
3576 				decoder->psb_ip = decoder->ip;  in intel_pt_walk_psb()
3579 							decoder->ip);  in intel_pt_walk_psb()
3584 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psb()
3588 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psb()
3592 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psb()
3596 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psb()
3600 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psb()
3604 			intel_pt_set_pip(decoder);  in intel_pt_walk_psb()
3608 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_psb()
3612 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psb()
3616 			decoder->pge = false;  in intel_pt_walk_psb()
3617 			decoder->continuous_period = false;  in intel_pt_walk_psb()
3618 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_psb()
3622 			decoder->have_tma = false;  in intel_pt_walk_psb()
3624 			if (decoder->ip)  in intel_pt_walk_psb()
3625 				decoder->pkt_state = INTEL_PT_STATE_ERR4;  in intel_pt_walk_psb()
3627 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_psb()
3632 			err = intel_pt_bug(decoder);  in intel_pt_walk_psb()
3636 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psb()
3652 	decoder->in_psb = false;  in intel_pt_walk_psb()
3657 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)  in intel_pt_walk_to_ip()  argument
3662 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_to_ip()
3666 		switch (decoder->packet.type) {  in intel_pt_walk_to_ip()
3668 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3669 			decoder->pge = false;  in intel_pt_walk_to_ip()
3670 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3671 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3672 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3674 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_to_ip()
3678 			decoder->pge = true;  in intel_pt_walk_to_ip()
3679 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_to_ip()
3680 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3681 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3682 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3684 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_to_ip()
3688 			decoder->pge = true;  in intel_pt_walk_to_ip()
3689 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3690 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3691 			if (!decoder->ip)  in intel_pt_walk_to_ip()
3696 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
3697 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
3698 			if (decoder->ip)  in intel_pt_walk_to_ip()
3703 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_to_ip()
3707 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_to_ip()
3711 			intel_pt_calc_tma(decoder);  in intel_pt_walk_to_ip()
3715 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_to_ip()
3719 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_to_ip()
3723 			intel_pt_set_pip(decoder);  in intel_pt_walk_to_ip()
3727 			intel_pt_mode_exec_status(decoder);  in intel_pt_walk_to_ip()
3731 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_to_ip()
3735 			return intel_pt_overflow(decoder);  in intel_pt_walk_to_ip()
3738 			return intel_pt_bug(decoder);  in intel_pt_walk_to_ip()
3741 			decoder->pge = false;  in intel_pt_walk_to_ip()
3742 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
3743 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_to_ip()
3744 			decoder->have_tma = false;  in intel_pt_walk_to_ip()
3748 			decoder->state.psb_offset = decoder->pos;  in intel_pt_walk_to_ip()
3749 			decoder->psb_ip = 0;  in intel_pt_walk_to_ip()
3750 			decoder->last_ip = 0;  in intel_pt_walk_to_ip()
3751 			decoder->have_last_ip = true;  in intel_pt_walk_to_ip()
3752 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_to_ip()
3753 			err = intel_pt_walk_psb(decoder);  in intel_pt_walk_to_ip()
3756 			decoder->state.type = INTEL_PT_PSB_EVT;  in intel_pt_walk_to_ip()
3757 			decoder->state.from_ip = decoder->psb_ip;  in intel_pt_walk_to_ip()
3758 			decoder->state.to_ip = 0;  in intel_pt_walk_to_ip()
3786 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)  in intel_pt_sync_ip()  argument
3790 	intel_pt_clear_fup_event(decoder);  in intel_pt_sync_ip()
3791 	decoder->overflow = false;  in intel_pt_sync_ip()
3793 	if (!decoder->branch_enable) {  in intel_pt_sync_ip()
3794 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3795 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync_ip()
3800 	err = intel_pt_walk_to_ip(decoder);  in intel_pt_sync_ip()
3801 	if (err || ((decoder->state.type & INTEL_PT_PSB_EVT) && !decoder->ip))  in intel_pt_sync_ip()
3805 	if (decoder->hop)  in intel_pt_sync_ip()
3806 		decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync_ip()
3808 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
3810 	decoder->state.from_ip = 0;  in intel_pt_sync_ip()
3811 	decoder->state.to_ip = decoder->ip;  in intel_pt_sync_ip()
3812 	intel_pt_log_to("Setting IP", decoder->ip);  in intel_pt_sync_ip()
3817 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)  in intel_pt_part_psb()  argument
3819 	const unsigned char *end = decoder->buf + decoder->len;  in intel_pt_part_psb()
3823 		if (i > decoder->len)  in intel_pt_part_psb()
3831 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)  in intel_pt_rest_psb()  argument
3836 	if (rest_psb > decoder->len ||  in intel_pt_rest_psb()
3837 	    memcmp(decoder->buf, psb + part_psb, rest_psb))  in intel_pt_rest_psb()
3843 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,  in intel_pt_get_split_psb()  argument
3848 	decoder->pos += decoder->len;  in intel_pt_get_split_psb()
3849 	decoder->len = 0;  in intel_pt_get_split_psb()
3851 	ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_split_psb()
3855 	rest_psb = intel_pt_rest_psb(decoder, part_psb);  in intel_pt_get_split_psb()
3859 	decoder->pos -= part_psb;  in intel_pt_get_split_psb()
3860 	decoder->next_buf = decoder->buf + rest_psb;  in intel_pt_get_split_psb()
3861 	decoder->next_len = decoder->len - rest_psb;  in intel_pt_get_split_psb()
3862 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);  in intel_pt_get_split_psb()
3863 	decoder->buf = decoder->temp_buf;  in intel_pt_get_split_psb()
3864 	decoder->len = INTEL_PT_PSB_LEN;  in intel_pt_get_split_psb()
3869 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)  in intel_pt_scan_for_psb()  argument
3876 		if (!decoder->len) {  in intel_pt_scan_for_psb()
3877 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_scan_for_psb()
3882 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,  in intel_pt_scan_for_psb()
3887 			part_psb = intel_pt_part_psb(decoder);  in intel_pt_scan_for_psb()
3889 				ret = intel_pt_get_split_psb(decoder, part_psb);  in intel_pt_scan_for_psb()
3893 				decoder->pos += decoder->len;  in intel_pt_scan_for_psb()
3894 				decoder->len = 0;  in intel_pt_scan_for_psb()
3899 		decoder->pkt_step = next - decoder->buf;  in intel_pt_scan_for_psb()
3900 		return intel_pt_get_next_packet(decoder);  in intel_pt_scan_for_psb()
3904 static int intel_pt_sync(struct intel_pt_decoder *decoder)  in intel_pt_sync()  argument
3908 	decoder->pge = false;  in intel_pt_sync()
3909 	decoder->continuous_period = false;  in intel_pt_sync()
3910 	decoder->have_last_ip = false;  in intel_pt_sync()
3911 	decoder->last_ip = 0;  in intel_pt_sync()
3912 	decoder->psb_ip = 0;  in intel_pt_sync()
3913 	decoder->ip = 0;  in intel_pt_sync()
3914 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_sync()
3916 	err = intel_pt_scan_for_psb(decoder);  in intel_pt_sync()
3920 	if (decoder->vm_time_correlation) {  in intel_pt_sync()
3921 		decoder->in_psb = true;  in intel_pt_sync()
3922 		if (!decoder->timestamp)  in intel_pt_sync()
3923 			decoder->timestamp = 1;  in intel_pt_sync()
3924 		decoder->state.type = 0;  in intel_pt_sync()
3925 		decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION;  in intel_pt_sync()
3929 	decoder->have_last_ip = true;  in intel_pt_sync()
3930 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
3932 	err = intel_pt_walk_psb(decoder);  in intel_pt_sync()
3936 	decoder->state.type = INTEL_PT_PSB_EVT; /* Only PSB sample */  in intel_pt_sync()
3937 	decoder->state.from_ip = decoder->psb_ip;  in intel_pt_sync()
3938 	decoder->state.to_ip = 0;  in intel_pt_sync()
3940 	if (decoder->ip) {  in intel_pt_sync()
3945 		if (decoder->hop)  in intel_pt_sync()
3946 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync()
3948 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
3954 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_est_timestamp()  argument
3956 	uint64_t est = decoder->sample_insn_cnt << 1;  in intel_pt_est_timestamp()
3958 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)  in intel_pt_est_timestamp()
3961 	est *= decoder->max_non_turbo_ratio;  in intel_pt_est_timestamp()
3962 	est /= decoder->cbr;  in intel_pt_est_timestamp()
3964 	return decoder->sample_timestamp + est;  in intel_pt_est_timestamp()
3967 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)  in intel_pt_decode()  argument
3972 		decoder->state.type = INTEL_PT_BRANCH;  in intel_pt_decode()
3973 		decoder->state.flags = 0;  in intel_pt_decode()
3975 		switch (decoder->pkt_state) {  in intel_pt_decode()
3977 			err = intel_pt_sync(decoder);  in intel_pt_decode()
3980 			decoder->have_last_ip = false;  in intel_pt_decode()
3981 			decoder->last_ip = 0;  in intel_pt_decode()
3982 			decoder->ip = 0;  in intel_pt_decode()
3985 			err = intel_pt_sync_ip(decoder);  in intel_pt_decode()
3988 			err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3992 			err = intel_pt_walk_tnt(decoder);  in intel_pt_decode()
3994 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
3998 			err = intel_pt_walk_tip(decoder);  in intel_pt_decode()
4001 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
4003 				err = intel_pt_walk_fup_tip(decoder);  in intel_pt_decode()
4006 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
4008 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
4011 			err = intel_pt_fup_in_psb(decoder);  in intel_pt_decode()
4014 			err = intel_pt_resample(decoder);  in intel_pt_decode()
4017 			err = intel_pt_vm_time_correlation(decoder);  in intel_pt_decode()
4020 			err = intel_pt_bug(decoder);  in intel_pt_decode()
4026 		decoder->state.err = intel_pt_ext_err(err);  in intel_pt_decode()
4028 			decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
4029 		intel_pt_update_sample_time(decoder);  in intel_pt_decode()
4030 		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
4031 		intel_pt_set_nr(decoder);  in intel_pt_decode()
4033 		decoder->state.err = 0;  in intel_pt_decode()
4034 		if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_decode()
4035 			decoder->cbr_seen = decoder->cbr;  in intel_pt_decode()
4036 			if (!decoder->state.type) {  in intel_pt_decode()
4037 				decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
4038 				decoder->state.to_ip = 0;  in intel_pt_decode()
4040 			decoder->state.type |= INTEL_PT_CBR_CHG;  in intel_pt_decode()
4041 			decoder->state.cbr_payload = decoder->cbr_payload;  in intel_pt_decode()
4042 			decoder->state.cbr = decoder->cbr;  in intel_pt_decode()
4044 		if (intel_pt_sample_time(decoder->pkt_state)) {  in intel_pt_decode()
4045 			intel_pt_update_sample_time(decoder);  in intel_pt_decode()
4046 			if (decoder->sample_cyc) {  in intel_pt_decode()
4047 				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
4048 				decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
4049 				decoder->sample_cyc = false;  in intel_pt_decode()
4056 		if (!decoder->have_cyc)  in intel_pt_decode()
4057 			decoder->state.flags |= INTEL_PT_SAMPLE_IPC;  in intel_pt_decode()
4061 	if ((decoder->state.type & INTEL_PT_PSB_EVT) && decoder->tsc_timestamp)  in intel_pt_decode()
4062 		decoder->sample_timestamp = decoder->tsc_timestamp;  in intel_pt_decode()
4064 	decoder->state.from_nr = decoder->nr;  in intel_pt_decode()
4065 	decoder->state.to_nr = decoder->next_nr;  in intel_pt_decode()
4066 	decoder->nr = decoder->next_nr;  in intel_pt_decode()
4068 	decoder->state.timestamp = decoder->sample_timestamp;  in intel_pt_decode()
4069 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);  in intel_pt_decode()
4070 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;  in intel_pt_decode()
4071 	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;  in intel_pt_decode()
4073 	return &decoder->state;  in intel_pt_decode()
4441  * intel_pt_fast_forward - reposition decoder forwards.
4442  * @decoder: Intel PT decoder
4445  * Reposition decoder at the last PSB with a timestamp earlier than @timestamp.
4449 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)  in intel_pt_fast_forward()  argument
4459 	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);  in intel_pt_fast_forward()
4466 			decoder->pos += decoder->len;  in intel_pt_fast_forward()
4467 			decoder->len = 0;  in intel_pt_fast_forward()
4468 			err = intel_pt_get_next_data(decoder, true);  in intel_pt_fast_forward()
4472 		} while (decoder->buf_timestamp != d.buf_timestamp);  in intel_pt_fast_forward()
4475 	if (!decoder->buf)  in intel_pt_fast_forward()
4478 	buf = (unsigned char *)decoder->buf;  in intel_pt_fast_forward()
4479 	len = decoder->len;  in intel_pt_fast_forward()
4494 		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);  in intel_pt_fast_forward()
4503 			decoder->pos += decoder->len - len;  in intel_pt_fast_forward()
4504 			decoder->buf = buf;  in intel_pt_fast_forward()
4505 			decoder->len = len;  in intel_pt_fast_forward()
4506 			intel_pt_reposition(decoder);  in intel_pt_fast_forward()