92 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
194 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)  in intel_pt_setup_period()  argument
196 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {  in intel_pt_setup_period()
199 		period = intel_pt_lower_power_of_2(decoder->period);  in intel_pt_setup_period()
200 		decoder->period_mask  = ~(period - 1);  in intel_pt_setup_period()
201 		decoder->period_ticks = period;  in intel_pt_setup_period()
214 	struct intel_pt_decoder *decoder;  in intel_pt_decoder_new()  local
219 	decoder = zalloc(sizeof(struct intel_pt_decoder));  in intel_pt_decoder_new()
220 	if (!decoder)  in intel_pt_decoder_new()
223 	decoder->get_trace          = params->get_trace;  in intel_pt_decoder_new()
224 	decoder->walk_insn          = params->walk_insn;  in intel_pt_decoder_new()
225 	decoder->pgd_ip             = params->pgd_ip;  in intel_pt_decoder_new()
226 	decoder->data               = params->data;  in intel_pt_decoder_new()
227 	decoder->return_compression = params->return_compression;  in intel_pt_decoder_new()
228 	decoder->branch_enable      = params->branch_enable;  in intel_pt_decoder_new()
230 	decoder->flags              = params->flags;  in intel_pt_decoder_new()
232 	decoder->period             = params->period;  in intel_pt_decoder_new()
233 	decoder->period_type        = params->period_type;  in intel_pt_decoder_new()
235 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
236 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
238 	intel_pt_setup_period(decoder);  in intel_pt_decoder_new()
240 	decoder->mtc_shift = params->mtc_period;  in intel_pt_decoder_new()
241 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;  in intel_pt_decoder_new()
243 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;  in intel_pt_decoder_new()
244 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
246 	if (!decoder->tsc_ctc_ratio_n)  in intel_pt_decoder_new()
247 		decoder->tsc_ctc_ratio_d = 0;  in intel_pt_decoder_new()
249 	if (decoder->tsc_ctc_ratio_d) {  in intel_pt_decoder_new()
250 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))  in intel_pt_decoder_new()
251 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /  in intel_pt_decoder_new()
252 						decoder->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
259 		decoder->tsc_slip = multdiv(2 << decoder->mtc_shift,  in intel_pt_decoder_new()
260 					decoder->tsc_ctc_ratio_n,  in intel_pt_decoder_new()
261 					decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
264 	if (decoder->tsc_slip < 0x100)  in intel_pt_decoder_new()
265 		decoder->tsc_slip = 0x100;  in intel_pt_decoder_new()
267 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);  in intel_pt_decoder_new()
268 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);  in intel_pt_decoder_new()
269 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
270 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);  in intel_pt_decoder_new()
271 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);  in intel_pt_decoder_new()
273 	return decoder;  in intel_pt_decoder_new()
347 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)  in intel_pt_decoder_free()  argument
349 	intel_pt_free_stack(&decoder->stack);  in intel_pt_decoder_free()
350 	free(decoder);  in intel_pt_decoder_free()
434 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_last_ip()  argument
436 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);  in intel_pt_set_last_ip()
437 	decoder->have_last_ip = true;  in intel_pt_set_last_ip()
440 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_ip()  argument
442 	intel_pt_set_last_ip(decoder);  in intel_pt_set_ip()
443 	decoder->ip = decoder->last_ip;  in intel_pt_set_ip()
446 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)  in intel_pt_decoder_log_packet()  argument
448 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,  in intel_pt_decoder_log_packet()
449 			    decoder->buf);  in intel_pt_decoder_log_packet()
452 static int intel_pt_bug(struct intel_pt_decoder *decoder)  in intel_pt_bug()  argument
455 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_bug()
459 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)  in intel_pt_clear_tx_flags()  argument
461 	decoder->tx_flags = 0;  in intel_pt_clear_tx_flags()
464 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)  in intel_pt_update_in_tx()  argument
466 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;  in intel_pt_update_in_tx()
469 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)  in intel_pt_bad_packet()  argument
471 	intel_pt_clear_tx_flags(decoder);  in intel_pt_bad_packet()
472 	decoder->have_tma = false;  in intel_pt_bad_packet()
473 	decoder->pkt_len = 1;  in intel_pt_bad_packet()
474 	decoder->pkt_step = 1;  in intel_pt_bad_packet()
475 	intel_pt_decoder_log_packet(decoder);  in intel_pt_bad_packet()
476 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {  in intel_pt_bad_packet()
478 		decoder->pkt_state = INTEL_PT_STATE_ERR1;  in intel_pt_bad_packet()
483 static int intel_pt_get_data(struct intel_pt_decoder *decoder)  in intel_pt_get_data()  argument
488 	decoder->pkt_step = 0;  in intel_pt_get_data()
491 	ret = decoder->get_trace(&buffer, decoder->data);  in intel_pt_get_data()
494 	decoder->buf = buffer.buf;  in intel_pt_get_data()
495 	decoder->len = buffer.len;  in intel_pt_get_data()
496 	if (!decoder->len) {  in intel_pt_get_data()
501 		decoder->ip = 0;  in intel_pt_get_data()
502 		decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_get_data()
503 		decoder->ref_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
504 		decoder->timestamp = 0;  in intel_pt_get_data()
505 		decoder->have_tma = false;  in intel_pt_get_data()
506 		decoder->state.trace_nr = buffer.trace_nr;  in intel_pt_get_data()
508 			     decoder->ref_timestamp);  in intel_pt_get_data()
515 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)  in intel_pt_get_next_data()  argument
517 	if (!decoder->next_buf)  in intel_pt_get_next_data()
518 		return intel_pt_get_data(decoder);  in intel_pt_get_next_data()
520 	decoder->buf = decoder->next_buf;  in intel_pt_get_next_data()
521 	decoder->len = decoder->next_len;  in intel_pt_get_next_data()
522 	decoder->next_buf = 0;  in intel_pt_get_next_data()
523 	decoder->next_len = 0;  in intel_pt_get_next_data()
527 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_split_packet()  argument
529 	unsigned char *buf = decoder->temp_buf;  in intel_pt_get_split_packet()
533 	old_len = decoder->len;  in intel_pt_get_split_packet()
534 	len = decoder->len;  in intel_pt_get_split_packet()
535 	memcpy(buf, decoder->buf, len);  in intel_pt_get_split_packet()
537 	ret = intel_pt_get_data(decoder);  in intel_pt_get_split_packet()
539 		decoder->pos += old_len;  in intel_pt_get_split_packet()
544 	if (n > decoder->len)  in intel_pt_get_split_packet()
545 		n = decoder->len;  in intel_pt_get_split_packet()
546 	memcpy(buf + len, decoder->buf, n);  in intel_pt_get_split_packet()
549 	ret = intel_pt_get_packet(buf, len, &decoder->packet);  in intel_pt_get_split_packet()
551 		decoder->next_buf = decoder->buf;  in intel_pt_get_split_packet()
552 		decoder->next_len = decoder->len;  in intel_pt_get_split_packet()
553 		decoder->buf = buf;  in intel_pt_get_split_packet()
554 		decoder->len = old_len;  in intel_pt_get_split_packet()
555 		return intel_pt_bad_packet(decoder);  in intel_pt_get_split_packet()
558 	decoder->next_buf = decoder->buf + (ret - old_len);  in intel_pt_get_split_packet()
559 	decoder->next_len = decoder->len - (ret - old_len);  in intel_pt_get_split_packet()
561 	decoder->buf = buf;  in intel_pt_get_split_packet()
562 	decoder->len = ret;  in intel_pt_get_split_packet()
568 	struct intel_pt_decoder	  *decoder;  member
579 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,  in intel_pt_pkt_lookahead()  argument
583 	const unsigned char *buf = decoder->buf;  in intel_pt_pkt_lookahead()
584 	size_t len = decoder->len;  in intel_pt_pkt_lookahead()
587 	pkt_info.decoder          = decoder;  in intel_pt_pkt_lookahead()
588 	pkt_info.pos              = decoder->pos;  in intel_pt_pkt_lookahead()
589 	pkt_info.pkt_len          = decoder->pkt_step;  in intel_pt_pkt_lookahead()
590 	pkt_info.last_packet_type = decoder->last_packet_type;  in intel_pt_pkt_lookahead()
655 	struct intel_pt_decoder *decoder = pkt_info->decoder;  in intel_pt_calc_cyc_cb()  local
684 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {  in intel_pt_calc_cyc_cb()
686 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_cyc_cb()
693 		data->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_cyc_cb()
696 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
698 				data->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
702 					decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
703 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
726 		    data->timestamp - timestamp < decoder->tsc_slip)  in intel_pt_calc_cyc_cb()
743 		if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_cyc_cb()
748 		ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_cyc_cb()
750 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_cyc_cb()
753 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
754 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
757 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
758 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
776 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cyc_cb()
792 	if (!data->cbr && decoder->cbr) {  in intel_pt_calc_cyc_cb()
793 		data->cbr = decoder->cbr;  in intel_pt_calc_cyc_cb()
794 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_cb()
800 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;  in intel_pt_calc_cyc_cb()
809 	decoder->calc_cyc_to_tsc = cyc_to_tsc;  in intel_pt_calc_cyc_cb()
810 	decoder->have_calc_cyc_to_tsc = true;  in intel_pt_calc_cyc_cb()
823 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_cyc_to_tsc()  argument
829 		.last_mtc       = decoder->last_mtc,  in intel_pt_calc_cyc_to_tsc()
830 		.ctc_timestamp  = decoder->ctc_timestamp,  in intel_pt_calc_cyc_to_tsc()
831 		.ctc_delta      = decoder->ctc_delta,  in intel_pt_calc_cyc_to_tsc()
832 		.tsc_timestamp  = decoder->tsc_timestamp,  in intel_pt_calc_cyc_to_tsc()
833 		.timestamp      = decoder->timestamp,  in intel_pt_calc_cyc_to_tsc()
834 		.have_tma       = decoder->have_tma,  in intel_pt_calc_cyc_to_tsc()
835 		.fixup_last_mtc = decoder->fixup_last_mtc,  in intel_pt_calc_cyc_to_tsc()
848 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);  in intel_pt_calc_cyc_to_tsc()
851 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_next_packet()  argument
855 	decoder->last_packet_type = decoder->packet.type;  in intel_pt_get_next_packet()
858 		decoder->pos += decoder->pkt_step;  in intel_pt_get_next_packet()
859 		decoder->buf += decoder->pkt_step;  in intel_pt_get_next_packet()
860 		decoder->len -= decoder->pkt_step;  in intel_pt_get_next_packet()
862 		if (!decoder->len) {  in intel_pt_get_next_packet()
863 			ret = intel_pt_get_next_data(decoder);  in intel_pt_get_next_packet()
868 		ret = intel_pt_get_packet(decoder->buf, decoder->len,  in intel_pt_get_next_packet()
869 					  &decoder->packet);  in intel_pt_get_next_packet()
871 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {  in intel_pt_get_next_packet()
872 			ret = intel_pt_get_split_packet(decoder);  in intel_pt_get_next_packet()
877 			return intel_pt_bad_packet(decoder);  in intel_pt_get_next_packet()
879 		decoder->pkt_len = ret;  in intel_pt_get_next_packet()
880 		decoder->pkt_step = ret;  in intel_pt_get_next_packet()
881 		intel_pt_decoder_log_packet(decoder);  in intel_pt_get_next_packet()
882 	} while (decoder->packet.type == INTEL_PT_PAD);  in intel_pt_get_next_packet()
887 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)  in intel_pt_next_period()  argument
891 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_next_period()
892 	masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
893 	if (decoder->continuous_period) {  in intel_pt_next_period()
894 		if (masked_timestamp != decoder->last_masked_timestamp)  in intel_pt_next_period()
898 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
899 		if (masked_timestamp != decoder->last_masked_timestamp) {  in intel_pt_next_period()
900 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_next_period()
901 			decoder->continuous_period = true;  in intel_pt_next_period()
904 	return decoder->period_ticks - (timestamp - masked_timestamp);  in intel_pt_next_period()
907 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)  in intel_pt_next_sample()  argument
909 	switch (decoder->period_type) {  in intel_pt_next_sample()
911 		return decoder->period - decoder->period_insn_cnt;  in intel_pt_next_sample()
913 		return intel_pt_next_period(decoder);  in intel_pt_next_sample()
921 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_insn()  argument
925 	switch (decoder->period_type) {  in intel_pt_sample_insn()
927 		decoder->period_insn_cnt = 0;  in intel_pt_sample_insn()
930 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_sample_insn()
931 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_sample_insn()
932 		decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_sample_insn()
940 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_insn()
943 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,  in intel_pt_walk_insn()  argument
949 	if (!decoder->mtc_insn)  in intel_pt_walk_insn()
950 		decoder->mtc_insn = true;  in intel_pt_walk_insn()
952 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_walk_insn()
954 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,  in intel_pt_walk_insn()
955 				 max_insn_cnt, decoder->data);  in intel_pt_walk_insn()
957 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
958 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
959 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
960 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
963 		decoder->no_progress = 0;  in intel_pt_walk_insn()
964 		decoder->pkt_state = INTEL_PT_STATE_ERR2;  in intel_pt_walk_insn()
966 				decoder->ip);  in intel_pt_walk_insn()
972 	if (ip && decoder->ip == ip) {  in intel_pt_walk_insn()
978 		intel_pt_sample_insn(decoder);  in intel_pt_walk_insn()
981 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_insn()
982 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
983 		decoder->state.to_ip = 0;  in intel_pt_walk_insn()
984 		decoder->ip += intel_pt_insn->length;  in intel_pt_walk_insn()
993 			err = intel_pt_push(&decoder->stack, decoder->ip +  in intel_pt_walk_insn()
999 		decoder->ret_addr = intel_pt_pop(&decoder->stack);  in intel_pt_walk_insn()
1003 		int cnt = decoder->no_progress++;  in intel_pt_walk_insn()
1005 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1006 		decoder->ip += intel_pt_insn->length +  in intel_pt_walk_insn()
1008 		decoder->state.to_ip = decoder->ip;  in intel_pt_walk_insn()
1020 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1021 				decoder->stuck_ip_prd = 1;  in intel_pt_walk_insn()
1022 				decoder->stuck_ip_cnt = 1;  in intel_pt_walk_insn()
1024 				   decoder->state.to_ip == decoder->stuck_ip) {  in intel_pt_walk_insn()
1026 						decoder->state.to_ip);  in intel_pt_walk_insn()
1027 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_insn()
1030 			} else if (!--decoder->stuck_ip_cnt) {  in intel_pt_walk_insn()
1031 				decoder->stuck_ip_prd += 1;  in intel_pt_walk_insn()
1032 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;  in intel_pt_walk_insn()
1033 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1039 	decoder->no_progress = 0;  in intel_pt_walk_insn()
1041 	decoder->state.insn_op = intel_pt_insn->op;  in intel_pt_walk_insn()
1042 	decoder->state.insn_len = intel_pt_insn->length;  in intel_pt_walk_insn()
1043 	memcpy(decoder->state.insn, intel_pt_insn->buf,  in intel_pt_walk_insn()
1046 	if (decoder->tx_flags & INTEL_PT_IN_TX)  in intel_pt_walk_insn()
1047 		decoder->state.flags |= INTEL_PT_IN_TX;  in intel_pt_walk_insn()
1052 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)  in intel_pt_fup_event()  argument
1056 	if (decoder->set_fup_tx_flags) {  in intel_pt_fup_event()
1057 		decoder->set_fup_tx_flags = false;  in intel_pt_fup_event()
1058 		decoder->tx_flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1059 		decoder->state.type = INTEL_PT_TRANSACTION;  in intel_pt_fup_event()
1060 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1061 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1062 		decoder->state.flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1065 	if (decoder->set_fup_ptw) {  in intel_pt_fup_event()
1066 		decoder->set_fup_ptw = false;  in intel_pt_fup_event()
1067 		decoder->state.type = INTEL_PT_PTW;  in intel_pt_fup_event()
1068 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1069 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1070 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1071 		decoder->state.ptw_payload = decoder->fup_ptw_payload;  in intel_pt_fup_event()
1074 	if (decoder->set_fup_mwait) {  in intel_pt_fup_event()
1075 		decoder->set_fup_mwait = false;  in intel_pt_fup_event()
1076 		decoder->state.type = INTEL_PT_MWAIT_OP;  in intel_pt_fup_event()
1077 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1078 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1079 		decoder->state.mwait_payload = decoder->fup_mwait_payload;  in intel_pt_fup_event()
1082 	if (decoder->set_fup_pwre) {  in intel_pt_fup_event()
1083 		decoder->set_fup_pwre = false;  in intel_pt_fup_event()
1084 		decoder->state.type |= INTEL_PT_PWR_ENTRY;  in intel_pt_fup_event()
1085 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1086 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1087 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1088 		decoder->state.pwre_payload = decoder->fup_pwre_payload;  in intel_pt_fup_event()
1091 	if (decoder->set_fup_exstop) {  in intel_pt_fup_event()
1092 		decoder->set_fup_exstop = false;  in intel_pt_fup_event()
1093 		decoder->state.type |= INTEL_PT_EX_STOP;  in intel_pt_fup_event()
1094 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1095 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1096 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1097 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1103 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,  in intel_pt_fup_with_nlip()  argument
1107 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&  in intel_pt_fup_with_nlip()
1109 	       ip == decoder->ip + intel_pt_insn->length;  in intel_pt_fup_with_nlip()
1112 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup()  argument
1118 	ip = decoder->last_ip;  in intel_pt_walk_fup()
1121 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);  in intel_pt_walk_fup()
1125 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {  in intel_pt_walk_fup()
1126 			if (intel_pt_fup_event(decoder))  in intel_pt_walk_fup()
1130 		decoder->set_fup_tx_flags = false;  in intel_pt_walk_fup()
1136 					decoder->ip);  in intel_pt_walk_fup()
1137 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1143 					decoder->ip);  in intel_pt_walk_fup()
1144 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1148 		intel_pt_bug(decoder);  in intel_pt_walk_fup()
1152 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_tip()  argument
1157 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tip()
1159 	    decoder->pgd_ip &&  in intel_pt_walk_tip()
1160 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1161 	    (decoder->state.type & INTEL_PT_BRANCH) &&  in intel_pt_walk_tip()
1162 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {  in intel_pt_walk_tip()
1164 		decoder->no_progress = 0;  in intel_pt_walk_tip()
1165 		decoder->pge = false;  in intel_pt_walk_tip()
1166 		decoder->continuous_period = false;  in intel_pt_walk_tip()
1167 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1168 		decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1177 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {  in intel_pt_walk_tip()
1178 			decoder->pge = false;  in intel_pt_walk_tip()
1179 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1180 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1181 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1182 			decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1183 			if (decoder->packet.count != 0)  in intel_pt_walk_tip()
1184 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1186 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1187 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1188 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1189 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1191 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1192 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1199 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +  in intel_pt_walk_tip()
1202 		if (decoder->pgd_ip &&  in intel_pt_walk_tip()
1203 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1204 		    decoder->pgd_ip(to_ip, decoder->data)) {  in intel_pt_walk_tip()
1206 			decoder->pge = false;  in intel_pt_walk_tip()
1207 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1208 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1209 			decoder->ip = to_ip;  in intel_pt_walk_tip()
1210 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1211 			decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1215 				decoder->ip);  in intel_pt_walk_tip()
1216 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_tip()
1220 	return intel_pt_bug(decoder);  in intel_pt_walk_tip()
1223 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)  in intel_pt_walk_tnt()  argument
1229 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tnt()
1236 			if (!decoder->return_compression) {  in intel_pt_walk_tnt()
1238 						decoder->ip);  in intel_pt_walk_tnt()
1239 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1242 			if (!decoder->ret_addr) {  in intel_pt_walk_tnt()
1244 						decoder->ip);  in intel_pt_walk_tnt()
1245 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1248 			if (!(decoder->tnt.payload & BIT63)) {  in intel_pt_walk_tnt()
1250 						decoder->ip);  in intel_pt_walk_tnt()
1251 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1254 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1255 			if (!decoder->tnt.count)  in intel_pt_walk_tnt()
1256 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1257 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1258 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1259 			decoder->ip = decoder->ret_addr;  in intel_pt_walk_tnt()
1260 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1266 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_tnt()
1269 			if (decoder->packet.type != INTEL_PT_TIP ||  in intel_pt_walk_tnt()
1270 			    decoder->packet.count == 0) {  in intel_pt_walk_tnt()
1272 						decoder->ip);  in intel_pt_walk_tnt()
1273 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1274 				decoder->pkt_step = 0;  in intel_pt_walk_tnt()
1277 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_tnt()
1278 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1279 			decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tnt()
1280 			decoder->ip = decoder->last_ip;  in intel_pt_walk_tnt()
1285 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1286 			if (!decoder->tnt.count)  in intel_pt_walk_tnt()
1287 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1288 			if (decoder->tnt.payload & BIT63) {  in intel_pt_walk_tnt()
1289 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1290 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1291 				decoder->ip += intel_pt_insn.length +  in intel_pt_walk_tnt()
1293 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1297 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {  in intel_pt_walk_tnt()
1298 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1299 				decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_tnt()
1300 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1301 				decoder->state.to_ip = 0;  in intel_pt_walk_tnt()
1302 				decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1305 			decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1306 			if (!decoder->tnt.count)  in intel_pt_walk_tnt()
1308 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1312 		return intel_pt_bug(decoder);  in intel_pt_walk_tnt()
1316 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)  in intel_pt_mode_tsx()  argument
1321 	fup_tx_flags = decoder->packet.payload &  in intel_pt_mode_tsx()
1323 	err = intel_pt_get_next_packet(decoder);  in intel_pt_mode_tsx()
1326 	if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_mode_tsx()
1327 		decoder->fup_tx_flags = fup_tx_flags;  in intel_pt_mode_tsx()
1328 		decoder->set_fup_tx_flags = true;  in intel_pt_mode_tsx()
1329 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))  in intel_pt_mode_tsx()
1333 				decoder->pos);  in intel_pt_mode_tsx()
1334 		intel_pt_update_in_tx(decoder);  in intel_pt_mode_tsx()
1339 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_tsc_timestamp()  argument
1343 	decoder->have_tma = false;  in intel_pt_calc_tsc_timestamp()
1345 	if (decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1346 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1347 			    (decoder->ref_timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1348 		if (timestamp < decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1349 			if (decoder->ref_timestamp - timestamp > (1ULL << 55))  in intel_pt_calc_tsc_timestamp()
1352 			if (timestamp - decoder->ref_timestamp > (1ULL << 55))  in intel_pt_calc_tsc_timestamp()
1355 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1356 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1357 		decoder->ref_timestamp = 0;  in intel_pt_calc_tsc_timestamp()
1358 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1359 	} else if (decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1360 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1361 			    (decoder->timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1362 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1363 		if (timestamp < decoder->timestamp &&  in intel_pt_calc_tsc_timestamp()
1364 		    decoder->timestamp - timestamp < decoder->tsc_slip) {  in intel_pt_calc_tsc_timestamp()
1367 			timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1369 		if (timestamp < decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1372 			decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1374 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1375 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1378 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_tsc_timestamp()
1379 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1380 		decoder->cycle_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1381 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_tsc_timestamp()
1382 		intel_pt_calc_cyc_to_tsc(decoder, false);  in intel_pt_calc_tsc_timestamp()
1385 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_tsc_timestamp()
1388 static int intel_pt_overflow(struct intel_pt_decoder *decoder)  in intel_pt_overflow()  argument
1391 	intel_pt_clear_tx_flags(decoder);  in intel_pt_overflow()
1392 	decoder->cbr = 0;  in intel_pt_overflow()
1393 	decoder->timestamp_insn_cnt = 0;  in intel_pt_overflow()
1394 	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_overflow()
1395 	decoder->overflow = true;  in intel_pt_overflow()
1399 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)  in intel_pt_calc_tma()  argument
1401 	uint32_t ctc = decoder->packet.payload;  in intel_pt_calc_tma()
1402 	uint32_t fc = decoder->packet.count;  in intel_pt_calc_tma()
1403 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_tma()
1405 	if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_tma()
1408 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_tma()
1409 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;  in intel_pt_calc_tma()
1410 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_tma()
1411 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_tma()
1413 		decoder->ctc_timestamp -= multdiv(ctc_rem,  in intel_pt_calc_tma()
1414 						  decoder->tsc_ctc_ratio_n,  in intel_pt_calc_tma()
1415 						  decoder->tsc_ctc_ratio_d);  in intel_pt_calc_tma()
1417 	decoder->ctc_delta = 0;  in intel_pt_calc_tma()
1418 	decoder->have_tma = true;  in intel_pt_calc_tma()
1419 	decoder->fixup_last_mtc = true;  in intel_pt_calc_tma()
1421 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);  in intel_pt_calc_tma()
1424 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_mtc_timestamp()  argument
1429 	if (!decoder->have_tma)  in intel_pt_calc_mtc_timestamp()
1432 	mtc = decoder->packet.payload;  in intel_pt_calc_mtc_timestamp()
1434 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {  in intel_pt_calc_mtc_timestamp()
1435 		decoder->fixup_last_mtc = false;  in intel_pt_calc_mtc_timestamp()
1436 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_mtc_timestamp()
1437 					&decoder->last_mtc);  in intel_pt_calc_mtc_timestamp()
1440 	if (mtc > decoder->last_mtc)  in intel_pt_calc_mtc_timestamp()
1441 		mtc_delta = mtc - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1443 		mtc_delta = mtc + 256 - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1445 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_mtc_timestamp()
1447 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_mtc_timestamp()
1448 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1449 			    decoder->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_mtc_timestamp()
1451 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1452 			    multdiv(decoder->ctc_delta,  in intel_pt_calc_mtc_timestamp()
1453 				    decoder->tsc_ctc_ratio_n,  in intel_pt_calc_mtc_timestamp()
1454 				    decoder->tsc_ctc_ratio_d);  in intel_pt_calc_mtc_timestamp()
1457 	if (timestamp < decoder->timestamp)  in intel_pt_calc_mtc_timestamp()
1459 			     timestamp, decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1461 		decoder->timestamp = timestamp;  in intel_pt_calc_mtc_timestamp()
1463 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1464 	decoder->last_mtc = mtc;  in intel_pt_calc_mtc_timestamp()
1466 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_mtc_timestamp()
1467 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_mtc_timestamp()
1468 		decoder->cycle_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1469 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_mtc_timestamp()
1470 		intel_pt_calc_cyc_to_tsc(decoder, true);  in intel_pt_calc_mtc_timestamp()
1474 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)  in intel_pt_calc_cbr()  argument
1476 	unsigned int cbr = decoder->packet.payload & 0xff;  in intel_pt_calc_cbr()
1478 	decoder->cbr_payload = decoder->packet.payload;  in intel_pt_calc_cbr()
1480 	if (decoder->cbr == cbr)  in intel_pt_calc_cbr()
1483 	decoder->cbr = cbr;  in intel_pt_calc_cbr()
1484 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cbr()
1487 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_cyc_timestamp()  argument
1489 	uint64_t timestamp = decoder->cyc_ref_timestamp;  in intel_pt_calc_cyc_timestamp()
1491 	decoder->have_cyc = true;  in intel_pt_calc_cyc_timestamp()
1493 	decoder->cycle_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
1495 	if (!decoder->cyc_ref_timestamp)  in intel_pt_calc_cyc_timestamp()
1498 	if (decoder->have_calc_cyc_to_tsc)  in intel_pt_calc_cyc_timestamp()
1499 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1500 	else if (decoder->cbr)  in intel_pt_calc_cyc_timestamp()
1501 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1505 	if (timestamp < decoder->timestamp)  in intel_pt_calc_cyc_timestamp()
1507 			     timestamp, decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
1509 		decoder->timestamp = timestamp;  in intel_pt_calc_cyc_timestamp()
1511 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_cyc_timestamp()
1515 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)  in intel_pt_walk_psbend()  argument
1520 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psbend()
1524 		switch (decoder->packet.type) {  in intel_pt_walk_psbend()
1542 			decoder->have_tma = false;  in intel_pt_walk_psbend()
1547 			return intel_pt_overflow(decoder);  in intel_pt_walk_psbend()
1550 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psbend()
1554 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psbend()
1558 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psbend()
1562 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psbend()
1566 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_psbend()
1570 			decoder->pge = true;  in intel_pt_walk_psbend()
1571 			if (decoder->packet.count)  in intel_pt_walk_psbend()
1572 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_psbend()
1576 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psbend()
1580 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psbend()
1581 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_psbend()
1582 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_psbend()
1595 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup_tip()  argument
1599 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {  in intel_pt_walk_fup_tip()
1600 		decoder->tx_flags = 0;  in intel_pt_walk_fup_tip()
1601 		decoder->state.flags &= ~INTEL_PT_IN_TX;  in intel_pt_walk_fup_tip()
1602 		decoder->state.flags |= INTEL_PT_ABORT_TX;  in intel_pt_walk_fup_tip()
1604 		decoder->state.flags |= INTEL_PT_ASYNC;  in intel_pt_walk_fup_tip()
1608 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_fup_tip()
1612 		switch (decoder->packet.type) {  in intel_pt_walk_fup_tip()
1630 			decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_fup_tip()
1631 			decoder->pkt_step = 0;  in intel_pt_walk_fup_tip()
1635 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_fup_tip()
1639 			return intel_pt_overflow(decoder);  in intel_pt_walk_fup_tip()
1642 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1643 			decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1644 			if (decoder->packet.count != 0) {  in intel_pt_walk_fup_tip()
1645 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1647 					     decoder->ip);  in intel_pt_walk_fup_tip()
1649 			decoder->pge = false;  in intel_pt_walk_fup_tip()
1650 			decoder->continuous_period = false;  in intel_pt_walk_fup_tip()
1654 			decoder->pge = true;  in intel_pt_walk_fup_tip()
1656 				     decoder->ip);  in intel_pt_walk_fup_tip()
1657 			decoder->state.from_ip = 0;  in intel_pt_walk_fup_tip()
1658 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1659 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1661 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1662 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1667 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1668 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1669 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1671 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1672 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1677 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_fup_tip()
1681 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_fup_tip()
1682 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_fup_tip()
1683 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_fup_tip()
1687 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_fup_tip()
1691 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_fup_tip()
1700 			return intel_pt_bug(decoder);  in intel_pt_walk_fup_tip()
1705 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)  in intel_pt_walk_trace()  argument
1711 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
1715 		switch (decoder->packet.type) {  in intel_pt_walk_trace()
1717 			if (!decoder->packet.count)  in intel_pt_walk_trace()
1719 			decoder->tnt = decoder->packet;  in intel_pt_walk_trace()
1720 			decoder->pkt_state = INTEL_PT_STATE_TNT;  in intel_pt_walk_trace()
1721 			err = intel_pt_walk_tnt(decoder);  in intel_pt_walk_trace()
1727 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
1728 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
1729 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;  in intel_pt_walk_trace()
1730 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
1733 			decoder->pge = true;  in intel_pt_walk_trace()
1734 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
1736 						decoder->pos);  in intel_pt_walk_trace()
1739 			intel_pt_set_ip(decoder);  in intel_pt_walk_trace()
1740 			decoder->state.from_ip = 0;  in intel_pt_walk_trace()
1741 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_trace()
1746 			return intel_pt_overflow(decoder);  in intel_pt_walk_trace()
1749 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
1750 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
1751 			decoder->pkt_state = INTEL_PT_STATE_TIP;  in intel_pt_walk_trace()
1752 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
1755 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
1757 						decoder->pos);  in intel_pt_walk_trace()
1761 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
1762 			if (!decoder->branch_enable) {  in intel_pt_walk_trace()
1763 				decoder->ip = decoder->last_ip;  in intel_pt_walk_trace()
1764 				if (intel_pt_fup_event(decoder))  in intel_pt_walk_trace()
1769 			if (decoder->set_fup_mwait)  in intel_pt_walk_trace()
1771 			err = intel_pt_walk_fup(decoder);  in intel_pt_walk_trace()
1776 					decoder->pkt_state =  in intel_pt_walk_trace()
1779 					decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_walk_trace()
1786 			return intel_pt_walk_fup_tip(decoder);  in intel_pt_walk_trace()
1789 			decoder->pge = false;  in intel_pt_walk_trace()
1790 			decoder->continuous_period = false;  in intel_pt_walk_trace()
1791 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_trace()
1792 			decoder->have_tma = false;  in intel_pt_walk_trace()
1796 			decoder->last_ip = 0;  in intel_pt_walk_trace()
1797 			decoder->have_last_ip = true;  in intel_pt_walk_trace()
1798 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_trace()
1799 			err = intel_pt_walk_psbend(decoder);  in intel_pt_walk_trace()
1807 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_trace()
1811 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_trace()
1812 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)  in intel_pt_walk_trace()
1818 			if (!decoder->mtc_insn)  in intel_pt_walk_trace()
1820 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
1822 			if (!decoder->timestamp)  in intel_pt_walk_trace()
1824 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_trace()
1825 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1826 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1827 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
1831 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_trace()
1835 			intel_pt_calc_tma(decoder);  in intel_pt_walk_trace()
1839 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_trace()
1843 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_trace()
1844 			if (!decoder->branch_enable &&  in intel_pt_walk_trace()
1845 			    decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
1846 				decoder->cbr_seen = decoder->cbr;  in intel_pt_walk_trace()
1847 				decoder->state.type = INTEL_PT_CBR_CHG;  in intel_pt_walk_trace()
1848 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1849 				decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1850 				decoder->state.cbr_payload =  in intel_pt_walk_trace()
1851 							decoder->packet.payload;  in intel_pt_walk_trace()
1857 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_trace()
1862 			if (!decoder->pge) {  in intel_pt_walk_trace()
1863 				intel_pt_update_in_tx(decoder);  in intel_pt_walk_trace()
1866 			err = intel_pt_mode_tsx(decoder, &no_tip);  in intel_pt_walk_trace()
1872 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
1881 			decoder->fup_ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
1882 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
1885 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
1886 				decoder->set_fup_ptw = true;  in intel_pt_walk_trace()
1890 						decoder->pos);  in intel_pt_walk_trace()
1895 			decoder->state.type = INTEL_PT_PTW;  in intel_pt_walk_trace()
1896 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1897 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1898 			decoder->state.ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
1902 			decoder->fup_mwait_payload = decoder->packet.payload;  in intel_pt_walk_trace()
1903 			decoder->set_fup_mwait = true;  in intel_pt_walk_trace()
1907 			if (decoder->set_fup_mwait) {  in intel_pt_walk_trace()
1908 				decoder->fup_pwre_payload =  in intel_pt_walk_trace()
1909 							decoder->packet.payload;  in intel_pt_walk_trace()
1910 				decoder->set_fup_pwre = true;  in intel_pt_walk_trace()
1913 			decoder->state.type = INTEL_PT_PWR_ENTRY;  in intel_pt_walk_trace()
1914 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1915 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1916 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
1920 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
1923 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
1924 				decoder->set_fup_exstop = true;  in intel_pt_walk_trace()
1928 						decoder->pos);  in intel_pt_walk_trace()
1933 			decoder->state.type = INTEL_PT_EX_STOP;  in intel_pt_walk_trace()
1934 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1935 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1939 			decoder->state.type = INTEL_PT_PWR_EXIT;  in intel_pt_walk_trace()
1940 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
1941 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
1942 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
1946 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
1951 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)  in intel_pt_have_ip()  argument
1953 	return decoder->packet.count &&  in intel_pt_have_ip()
1954 	       (decoder->have_last_ip || decoder->packet.count == 3 ||  in intel_pt_have_ip()
1955 		decoder->packet.count == 6);  in intel_pt_have_ip()
1959 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)  in intel_pt_walk_psb()  argument
1964 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psb()
1968 		switch (decoder->packet.type) {  in intel_pt_walk_psb()
1970 			decoder->continuous_period = false;  in intel_pt_walk_psb()
1985 			decoder->pge = true;  in intel_pt_walk_psb()
1986 			if (intel_pt_have_ip(decoder)) {  in intel_pt_walk_psb()
1987 				uint64_t current_ip = decoder->ip;  in intel_pt_walk_psb()
1989 				intel_pt_set_ip(decoder);  in intel_pt_walk_psb()
1992 							decoder->ip);  in intel_pt_walk_psb()
1997 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psb()
2001 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psb()
2005 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psb()
2009 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psb()
2013 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psb()
2017 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_psb()
2021 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psb()
2025 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psb()
2029 			decoder->pge = false;  in intel_pt_walk_psb()
2030 			decoder->continuous_period = false;  in intel_pt_walk_psb()
2031 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_psb()
2035 			decoder->have_tma = false;  in intel_pt_walk_psb()
2037 			if (decoder->ip)  in intel_pt_walk_psb()
2038 				decoder->pkt_state = INTEL_PT_STATE_ERR4;  in intel_pt_walk_psb()
2040 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_psb()
2044 			return intel_pt_bug(decoder);  in intel_pt_walk_psb()
2047 			return intel_pt_overflow(decoder);  in intel_pt_walk_psb()
2062 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)  in intel_pt_walk_to_ip()  argument
2067 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_to_ip()
2071 		switch (decoder->packet.type) {  in intel_pt_walk_to_ip()
2073 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
2077 			decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;  in intel_pt_walk_to_ip()
2078 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2079 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2080 			if (decoder->ip)  in intel_pt_walk_to_ip()
2085 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2086 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2087 			if (decoder->ip)  in intel_pt_walk_to_ip()
2092 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_to_ip()
2096 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_to_ip()
2100 			intel_pt_calc_tma(decoder);  in intel_pt_walk_to_ip()
2104 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_to_ip()
2108 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_to_ip()
2112 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_to_ip()
2116 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_to_ip()
2120 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_to_ip()
2124 			return intel_pt_overflow(decoder);  in intel_pt_walk_to_ip()
2127 			return intel_pt_bug(decoder);  in intel_pt_walk_to_ip()
2130 			decoder->pge = false;  in intel_pt_walk_to_ip()
2131 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
2132 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_to_ip()
2133 			decoder->have_tma = false;  in intel_pt_walk_to_ip()
2137 			decoder->last_ip = 0;  in intel_pt_walk_to_ip()
2138 			decoder->have_last_ip = true;  in intel_pt_walk_to_ip()
2139 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_to_ip()
2140 			err = intel_pt_walk_psb(decoder);  in intel_pt_walk_to_ip()
2143 			if (decoder->ip) {  in intel_pt_walk_to_ip()
2145 				decoder->state.type = 0;  in intel_pt_walk_to_ip()
2168 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)  in intel_pt_sync_ip()  argument
2172 	decoder->set_fup_tx_flags = false;  in intel_pt_sync_ip()
2173 	decoder->set_fup_ptw = false;  in intel_pt_sync_ip()
2174 	decoder->set_fup_mwait = false;  in intel_pt_sync_ip()
2175 	decoder->set_fup_pwre = false;  in intel_pt_sync_ip()
2176 	decoder->set_fup_exstop = false;  in intel_pt_sync_ip()
2178 	if (!decoder->branch_enable) {  in intel_pt_sync_ip()
2179 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
2180 		decoder->overflow = false;  in intel_pt_sync_ip()
2181 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync_ip()
2186 	err = intel_pt_walk_to_ip(decoder);  in intel_pt_sync_ip()
2190 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
2191 	decoder->overflow = false;  in intel_pt_sync_ip()
2193 	decoder->state.from_ip = 0;  in intel_pt_sync_ip()
2194 	decoder->state.to_ip = decoder->ip;  in intel_pt_sync_ip()
2195 	intel_pt_log_to("Setting IP", decoder->ip);  in intel_pt_sync_ip()
2200 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)  in intel_pt_part_psb()  argument
2202 	const unsigned char *end = decoder->buf + decoder->len;  in intel_pt_part_psb()
2206 		if (i > decoder->len)  in intel_pt_part_psb()
2214 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)  in intel_pt_rest_psb()  argument
2219 	if (rest_psb > decoder->len ||  in intel_pt_rest_psb()
2220 	    memcmp(decoder->buf, psb + part_psb, rest_psb))  in intel_pt_rest_psb()
2226 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,  in intel_pt_get_split_psb()  argument
2231 	decoder->pos += decoder->len;  in intel_pt_get_split_psb()
2232 	decoder->len = 0;  in intel_pt_get_split_psb()
2234 	ret = intel_pt_get_next_data(decoder);  in intel_pt_get_split_psb()
2238 	rest_psb = intel_pt_rest_psb(decoder, part_psb);  in intel_pt_get_split_psb()
2242 	decoder->pos -= part_psb;  in intel_pt_get_split_psb()
2243 	decoder->next_buf = decoder->buf + rest_psb;  in intel_pt_get_split_psb()
2244 	decoder->next_len = decoder->len - rest_psb;  in intel_pt_get_split_psb()
2245 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);  in intel_pt_get_split_psb()
2246 	decoder->buf = decoder->temp_buf;  in intel_pt_get_split_psb()
2247 	decoder->len = INTEL_PT_PSB_LEN;  in intel_pt_get_split_psb()
2252 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)  in intel_pt_scan_for_psb()  argument
2259 		if (!decoder->len) {  in intel_pt_scan_for_psb()
2260 			ret = intel_pt_get_next_data(decoder);  in intel_pt_scan_for_psb()
2265 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,  in intel_pt_scan_for_psb()
2270 			part_psb = intel_pt_part_psb(decoder);  in intel_pt_scan_for_psb()
2272 				ret = intel_pt_get_split_psb(decoder, part_psb);  in intel_pt_scan_for_psb()
2276 				decoder->pos += decoder->len;  in intel_pt_scan_for_psb()
2277 				decoder->len = 0;  in intel_pt_scan_for_psb()
2282 		decoder->pkt_step = next - decoder->buf;  in intel_pt_scan_for_psb()
2283 		return intel_pt_get_next_packet(decoder);  in intel_pt_scan_for_psb()
2287 static int intel_pt_sync(struct intel_pt_decoder *decoder)  in intel_pt_sync()  argument
2291 	decoder->pge = false;  in intel_pt_sync()
2292 	decoder->continuous_period = false;  in intel_pt_sync()
2293 	decoder->have_last_ip = false;  in intel_pt_sync()
2294 	decoder->last_ip = 0;  in intel_pt_sync()
2295 	decoder->ip = 0;  in intel_pt_sync()
2296 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_sync()
2298 	err = intel_pt_scan_for_psb(decoder);  in intel_pt_sync()
2302 	decoder->have_last_ip = true;  in intel_pt_sync()
2303 	decoder->pkt_state = INTEL_PT_STATE_NO_IP;  in intel_pt_sync()
2305 	err = intel_pt_walk_psb(decoder);  in intel_pt_sync()
2309 	if (decoder->ip) {  in intel_pt_sync()
2310 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync()
2311 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
2313 		return intel_pt_sync_ip(decoder);  in intel_pt_sync()
2319 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_est_timestamp()  argument
2321 	uint64_t est = decoder->sample_insn_cnt << 1;  in intel_pt_est_timestamp()
2323 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)  in intel_pt_est_timestamp()
2326 	est *= decoder->max_non_turbo_ratio;  in intel_pt_est_timestamp()
2327 	est /= decoder->cbr;  in intel_pt_est_timestamp()
2329 	return decoder->sample_timestamp + est;  in intel_pt_est_timestamp()
2332 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)  in intel_pt_decode()  argument
2337 		decoder->state.type = INTEL_PT_BRANCH;  in intel_pt_decode()
2338 		decoder->state.flags = 0;  in intel_pt_decode()
2340 		switch (decoder->pkt_state) {  in intel_pt_decode()
2342 			err = intel_pt_sync(decoder);  in intel_pt_decode()
2345 			decoder->have_last_ip = false;  in intel_pt_decode()
2346 			decoder->last_ip = 0;  in intel_pt_decode()
2347 			decoder->ip = 0;  in intel_pt_decode()
2350 			err = intel_pt_sync_ip(decoder);  in intel_pt_decode()
2353 			err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2356 			err = intel_pt_walk_tnt(decoder);  in intel_pt_decode()
2358 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2362 			err = intel_pt_walk_tip(decoder);  in intel_pt_decode()
2365 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_decode()
2366 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
2368 				err = intel_pt_walk_fup_tip(decoder);  in intel_pt_decode()
2370 				decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_decode()
2373 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_decode()
2374 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
2376 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2379 			err = intel_pt_bug(decoder);  in intel_pt_decode()
2385 		decoder->state.err = intel_pt_ext_err(err);  in intel_pt_decode()
2386 		decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
2387 		decoder->sample_timestamp = decoder->timestamp;  in intel_pt_decode()
2388 		decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_decode()
2390 		decoder->state.err = 0;  in intel_pt_decode()
2391 		if (decoder->cbr != decoder->cbr_seen && decoder->state.type) {  in intel_pt_decode()
2392 			decoder->cbr_seen = decoder->cbr;  in intel_pt_decode()
2393 			decoder->state.type |= INTEL_PT_CBR_CHG;  in intel_pt_decode()
2394 			decoder->state.cbr_payload = decoder->cbr_payload;  in intel_pt_decode()
2396 		if (intel_pt_sample_time(decoder->pkt_state)) {  in intel_pt_decode()
2397 			decoder->sample_timestamp = decoder->timestamp;  in intel_pt_decode()
2398 			decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_decode()
2402 	decoder->state.timestamp = decoder->sample_timestamp;  in intel_pt_decode()
2403 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);  in intel_pt_decode()
2404 	decoder->state.cr3 = decoder->cr3;  in intel_pt_decode()
2405 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;  in intel_pt_decode()
2407 	return &decoder->state;  in intel_pt_decode()