88 #define INTEL_PT_STATE_ERR1	(decoder->pkt_state)
207 static void intel_pt_setup_period(struct intel_pt_decoder *decoder)  in intel_pt_setup_period()  argument
209 	if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {  in intel_pt_setup_period()
212 		period = intel_pt_lower_power_of_2(decoder->period);  in intel_pt_setup_period()
213 		decoder->period_mask  = ~(period - 1);  in intel_pt_setup_period()
214 		decoder->period_ticks = period;  in intel_pt_setup_period()
227 	struct intel_pt_decoder *decoder;  in intel_pt_decoder_new()  local
232 	decoder = zalloc(sizeof(struct intel_pt_decoder));  in intel_pt_decoder_new()
233 	if (!decoder)  in intel_pt_decoder_new()
236 	decoder->get_trace          = params->get_trace;  in intel_pt_decoder_new()
237 	decoder->walk_insn          = params->walk_insn;  in intel_pt_decoder_new()
238 	decoder->pgd_ip             = params->pgd_ip;  in intel_pt_decoder_new()
239 	decoder->lookahead          = params->lookahead;  in intel_pt_decoder_new()
240 	decoder->data               = params->data;  in intel_pt_decoder_new()
241 	decoder->return_compression = params->return_compression;  in intel_pt_decoder_new()
242 	decoder->branch_enable      = params->branch_enable;  in intel_pt_decoder_new()
243 	decoder->hop                = params->quick >= 1;  in intel_pt_decoder_new()
244 	decoder->leap               = params->quick >= 2;  in intel_pt_decoder_new()
246 	decoder->flags              = params->flags;  in intel_pt_decoder_new()
248 	decoder->period             = params->period;  in intel_pt_decoder_new()
249 	decoder->period_type        = params->period_type;  in intel_pt_decoder_new()
251 	decoder->max_non_turbo_ratio    = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
252 	decoder->max_non_turbo_ratio_fp = params->max_non_turbo_ratio;  in intel_pt_decoder_new()
254 	intel_pt_setup_period(decoder);  in intel_pt_decoder_new()
256 	decoder->mtc_shift = params->mtc_period;  in intel_pt_decoder_new()
257 	decoder->ctc_rem_mask = (1 << decoder->mtc_shift) - 1;  in intel_pt_decoder_new()
259 	decoder->tsc_ctc_ratio_n = params->tsc_ctc_ratio_n;  in intel_pt_decoder_new()
260 	decoder->tsc_ctc_ratio_d = params->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
262 	if (!decoder->tsc_ctc_ratio_n)  in intel_pt_decoder_new()
263 		decoder->tsc_ctc_ratio_d = 0;  in intel_pt_decoder_new()
265 	if (decoder->tsc_ctc_ratio_d) {  in intel_pt_decoder_new()
266 		if (!(decoder->tsc_ctc_ratio_n % decoder->tsc_ctc_ratio_d))  in intel_pt_decoder_new()
267 			decoder->tsc_ctc_mult = decoder->tsc_ctc_ratio_n /  in intel_pt_decoder_new()
268 						decoder->tsc_ctc_ratio_d;  in intel_pt_decoder_new()
277 	decoder->tsc_slip = 0x10000;  in intel_pt_decoder_new()
279 	intel_pt_log("timestamp: mtc_shift %u\n", decoder->mtc_shift);  in intel_pt_decoder_new()
280 	intel_pt_log("timestamp: tsc_ctc_ratio_n %u\n", decoder->tsc_ctc_ratio_n);  in intel_pt_decoder_new()
281 	intel_pt_log("timestamp: tsc_ctc_ratio_d %u\n", decoder->tsc_ctc_ratio_d);  in intel_pt_decoder_new()
282 	intel_pt_log("timestamp: tsc_ctc_mult %u\n", decoder->tsc_ctc_mult);  in intel_pt_decoder_new()
283 	intel_pt_log("timestamp: tsc_slip %#x\n", decoder->tsc_slip);  in intel_pt_decoder_new()
285 	if (decoder->hop)  in intel_pt_decoder_new()
288 	return decoder;  in intel_pt_decoder_new()
362 void intel_pt_decoder_free(struct intel_pt_decoder *decoder)  in intel_pt_decoder_free()  argument
364 	intel_pt_free_stack(&decoder->stack);  in intel_pt_decoder_free()
365 	free(decoder);  in intel_pt_decoder_free()
449 static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_last_ip()  argument
451 	decoder->last_ip = intel_pt_calc_ip(&decoder->packet, decoder->last_ip);  in intel_pt_set_last_ip()
452 	decoder->have_last_ip = true;  in intel_pt_set_last_ip()
455 static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)  in intel_pt_set_ip()  argument
457 	intel_pt_set_last_ip(decoder);  in intel_pt_set_ip()
458 	decoder->ip = decoder->last_ip;  in intel_pt_set_ip()
461 static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)  in intel_pt_decoder_log_packet()  argument
463 	intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,  in intel_pt_decoder_log_packet()
464 			    decoder->buf);  in intel_pt_decoder_log_packet()
467 static int intel_pt_bug(struct intel_pt_decoder *decoder)  in intel_pt_bug()  argument
470 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_bug()
474 static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)  in intel_pt_clear_tx_flags()  argument
476 	decoder->tx_flags = 0;  in intel_pt_clear_tx_flags()
479 static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)  in intel_pt_update_in_tx()  argument
481 	decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;  in intel_pt_update_in_tx()
484 static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)  in intel_pt_bad_packet()  argument
486 	intel_pt_clear_tx_flags(decoder);  in intel_pt_bad_packet()
487 	decoder->have_tma = false;  in intel_pt_bad_packet()
488 	decoder->pkt_len = 1;  in intel_pt_bad_packet()
489 	decoder->pkt_step = 1;  in intel_pt_bad_packet()
490 	intel_pt_decoder_log_packet(decoder);  in intel_pt_bad_packet()
491 	if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {  in intel_pt_bad_packet()
493 		decoder->pkt_state = INTEL_PT_STATE_ERR1;  in intel_pt_bad_packet()
498 static inline void intel_pt_update_sample_time(struct intel_pt_decoder *decoder)  in intel_pt_update_sample_time()  argument
500 	decoder->sample_timestamp = decoder->timestamp;  in intel_pt_update_sample_time()
501 	decoder->sample_insn_cnt = decoder->timestamp_insn_cnt;  in intel_pt_update_sample_time()
504 static void intel_pt_reposition(struct intel_pt_decoder *decoder)  in intel_pt_reposition()  argument
506 	decoder->ip = 0;  in intel_pt_reposition()
507 	decoder->pkt_state = INTEL_PT_STATE_NO_PSB;  in intel_pt_reposition()
508 	decoder->timestamp = 0;  in intel_pt_reposition()
509 	decoder->have_tma = false;  in intel_pt_reposition()
512 static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition)  in intel_pt_get_data()  argument
517 	decoder->pkt_step = 0;  in intel_pt_get_data()
520 	ret = decoder->get_trace(&buffer, decoder->data);  in intel_pt_get_data()
523 	decoder->buf = buffer.buf;  in intel_pt_get_data()
524 	decoder->len = buffer.len;  in intel_pt_get_data()
525 	if (!decoder->len) {  in intel_pt_get_data()
529 	decoder->buf_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
531 		intel_pt_reposition(decoder);  in intel_pt_get_data()
532 		decoder->ref_timestamp = buffer.ref_timestamp;  in intel_pt_get_data()
533 		decoder->state.trace_nr = buffer.trace_nr;  in intel_pt_get_data()
535 			     decoder->ref_timestamp);  in intel_pt_get_data()
542 static int intel_pt_get_next_data(struct intel_pt_decoder *decoder,  in intel_pt_get_next_data()  argument
545 	if (!decoder->next_buf)  in intel_pt_get_next_data()
546 		return intel_pt_get_data(decoder, reposition);  in intel_pt_get_next_data()
548 	decoder->buf = decoder->next_buf;  in intel_pt_get_next_data()
549 	decoder->len = decoder->next_len;  in intel_pt_get_next_data()
550 	decoder->next_buf = 0;  in intel_pt_get_next_data()
551 	decoder->next_len = 0;  in intel_pt_get_next_data()
555 static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_split_packet()  argument
557 	unsigned char *buf = decoder->temp_buf;  in intel_pt_get_split_packet()
561 	old_len = decoder->len;  in intel_pt_get_split_packet()
562 	len = decoder->len;  in intel_pt_get_split_packet()
563 	memcpy(buf, decoder->buf, len);  in intel_pt_get_split_packet()
565 	ret = intel_pt_get_data(decoder, false);  in intel_pt_get_split_packet()
567 		decoder->pos += old_len;  in intel_pt_get_split_packet()
572 	if (n > decoder->len)  in intel_pt_get_split_packet()
573 		n = decoder->len;  in intel_pt_get_split_packet()
574 	memcpy(buf + len, decoder->buf, n);  in intel_pt_get_split_packet()
577 	decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_split_packet()
578 	ret = intel_pt_get_packet(buf, len, &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_split_packet()
580 		decoder->next_buf = decoder->buf;  in intel_pt_get_split_packet()
581 		decoder->next_len = decoder->len;  in intel_pt_get_split_packet()
582 		decoder->buf = buf;  in intel_pt_get_split_packet()
583 		decoder->len = old_len;  in intel_pt_get_split_packet()
584 		return intel_pt_bad_packet(decoder);  in intel_pt_get_split_packet()
587 	decoder->next_buf = decoder->buf + (ret - old_len);  in intel_pt_get_split_packet()
588 	decoder->next_len = decoder->len - (ret - old_len);  in intel_pt_get_split_packet()
590 	decoder->buf = buf;  in intel_pt_get_split_packet()
591 	decoder->len = ret;  in intel_pt_get_split_packet()
597 	struct intel_pt_decoder	  *decoder;  member
608 static int intel_pt_pkt_lookahead(struct intel_pt_decoder *decoder,  in intel_pt_pkt_lookahead()  argument
612 	const unsigned char *buf = decoder->buf;  in intel_pt_pkt_lookahead()
613 	enum intel_pt_pkt_ctx pkt_ctx = decoder->pkt_ctx;  in intel_pt_pkt_lookahead()
614 	size_t len = decoder->len;  in intel_pt_pkt_lookahead()
617 	pkt_info.decoder          = decoder;  in intel_pt_pkt_lookahead()
618 	pkt_info.pos              = decoder->pos;  in intel_pt_pkt_lookahead()
619 	pkt_info.pkt_len          = decoder->pkt_step;  in intel_pt_pkt_lookahead()
620 	pkt_info.last_packet_type = decoder->last_packet_type;  in intel_pt_pkt_lookahead()
686 	struct intel_pt_decoder *decoder = pkt_info->decoder;  in intel_pt_calc_cyc_cb()  local
719 		if (decoder->mtc_shift > 8 && data->fixup_last_mtc) {  in intel_pt_calc_cyc_cb()
721 			intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_cyc_cb()
728 		data->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_cyc_cb()
731 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
733 				data->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
737 					decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
738 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
761 		    data->timestamp - timestamp < decoder->tsc_slip)  in intel_pt_calc_cyc_cb()
778 		if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_cyc_cb()
783 		ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_cyc_cb()
785 		data->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_cyc_cb()
788 		if (decoder->tsc_ctc_mult) {  in intel_pt_calc_cyc_cb()
789 			data->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_cyc_cb()
792 				multdiv(ctc_rem, decoder->tsc_ctc_ratio_n,  in intel_pt_calc_cyc_cb()
793 					decoder->tsc_ctc_ratio_d);  in intel_pt_calc_cyc_cb()
811 		data->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cyc_cb()
827 	if (!data->cbr && decoder->cbr) {  in intel_pt_calc_cyc_cb()
828 		data->cbr = decoder->cbr;  in intel_pt_calc_cyc_cb()
829 		data->cbr_cyc_to_tsc = decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_cb()
835 	cyc_to_tsc = (double)(timestamp - decoder->timestamp) / data->cycle_cnt;  in intel_pt_calc_cyc_cb()
844 	decoder->calc_cyc_to_tsc = cyc_to_tsc;  in intel_pt_calc_cyc_cb()
845 	decoder->have_calc_cyc_to_tsc = true;  in intel_pt_calc_cyc_cb()
858 static void intel_pt_calc_cyc_to_tsc(struct intel_pt_decoder *decoder,  in intel_pt_calc_cyc_to_tsc()  argument
864 		.last_mtc       = decoder->last_mtc,  in intel_pt_calc_cyc_to_tsc()
865 		.ctc_timestamp  = decoder->ctc_timestamp,  in intel_pt_calc_cyc_to_tsc()
866 		.ctc_delta      = decoder->ctc_delta,  in intel_pt_calc_cyc_to_tsc()
867 		.tsc_timestamp  = decoder->tsc_timestamp,  in intel_pt_calc_cyc_to_tsc()
868 		.timestamp      = decoder->timestamp,  in intel_pt_calc_cyc_to_tsc()
869 		.have_tma       = decoder->have_tma,  in intel_pt_calc_cyc_to_tsc()
870 		.fixup_last_mtc = decoder->fixup_last_mtc,  in intel_pt_calc_cyc_to_tsc()
883 	intel_pt_pkt_lookahead(decoder, intel_pt_calc_cyc_cb, &data);  in intel_pt_calc_cyc_to_tsc()
886 static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)  in intel_pt_get_next_packet()  argument
890 	decoder->last_packet_type = decoder->packet.type;  in intel_pt_get_next_packet()
893 		decoder->pos += decoder->pkt_step;  in intel_pt_get_next_packet()
894 		decoder->buf += decoder->pkt_step;  in intel_pt_get_next_packet()
895 		decoder->len -= decoder->pkt_step;  in intel_pt_get_next_packet()
897 		if (!decoder->len) {  in intel_pt_get_next_packet()
898 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_next_packet()
903 		decoder->prev_pkt_ctx = decoder->pkt_ctx;  in intel_pt_get_next_packet()
904 		ret = intel_pt_get_packet(decoder->buf, decoder->len,  in intel_pt_get_next_packet()
905 					  &decoder->packet, &decoder->pkt_ctx);  in intel_pt_get_next_packet()
907 		    decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {  in intel_pt_get_next_packet()
908 			ret = intel_pt_get_split_packet(decoder);  in intel_pt_get_next_packet()
913 			return intel_pt_bad_packet(decoder);  in intel_pt_get_next_packet()
915 		decoder->pkt_len = ret;  in intel_pt_get_next_packet()
916 		decoder->pkt_step = ret;  in intel_pt_get_next_packet()
917 		intel_pt_decoder_log_packet(decoder);  in intel_pt_get_next_packet()
918 	} while (decoder->packet.type == INTEL_PT_PAD);  in intel_pt_get_next_packet()
923 static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)  in intel_pt_next_period()  argument
927 	timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_next_period()
928 	masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
929 	if (decoder->continuous_period) {  in intel_pt_next_period()
930 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_next_period()
934 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_next_period()
935 		if (masked_timestamp > decoder->last_masked_timestamp) {  in intel_pt_next_period()
936 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_next_period()
937 			decoder->continuous_period = true;  in intel_pt_next_period()
941 	if (masked_timestamp < decoder->last_masked_timestamp)  in intel_pt_next_period()
942 		return decoder->period_ticks;  in intel_pt_next_period()
944 	return decoder->period_ticks - (timestamp - masked_timestamp);  in intel_pt_next_period()
947 static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)  in intel_pt_next_sample()  argument
949 	switch (decoder->period_type) {  in intel_pt_next_sample()
951 		return decoder->period - decoder->period_insn_cnt;  in intel_pt_next_sample()
953 		return intel_pt_next_period(decoder);  in intel_pt_next_sample()
961 static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)  in intel_pt_sample_insn()  argument
965 	switch (decoder->period_type) {  in intel_pt_sample_insn()
967 		decoder->period_insn_cnt = 0;  in intel_pt_sample_insn()
970 		timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;  in intel_pt_sample_insn()
971 		masked_timestamp = timestamp & decoder->period_mask;  in intel_pt_sample_insn()
972 		if (masked_timestamp > decoder->last_masked_timestamp)  in intel_pt_sample_insn()
973 			decoder->last_masked_timestamp = masked_timestamp;  in intel_pt_sample_insn()
975 			decoder->last_masked_timestamp += decoder->period_ticks;  in intel_pt_sample_insn()
983 	decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_sample_insn()
986 static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,  in intel_pt_walk_insn()  argument
992 	if (!decoder->mtc_insn)  in intel_pt_walk_insn()
993 		decoder->mtc_insn = true;  in intel_pt_walk_insn()
995 	max_insn_cnt = intel_pt_next_sample(decoder);  in intel_pt_walk_insn()
997 	err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,  in intel_pt_walk_insn()
998 				 max_insn_cnt, decoder->data);  in intel_pt_walk_insn()
1000 	decoder->tot_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1001 	decoder->timestamp_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1002 	decoder->sample_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1003 	decoder->period_insn_cnt += insn_cnt;  in intel_pt_walk_insn()
1006 		decoder->no_progress = 0;  in intel_pt_walk_insn()
1007 		decoder->pkt_state = INTEL_PT_STATE_ERR2;  in intel_pt_walk_insn()
1009 				decoder->ip);  in intel_pt_walk_insn()
1015 	if (ip && decoder->ip == ip) {  in intel_pt_walk_insn()
1021 		intel_pt_sample_insn(decoder);  in intel_pt_walk_insn()
1024 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_insn()
1025 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1026 		decoder->state.to_ip = 0;  in intel_pt_walk_insn()
1027 		decoder->ip += intel_pt_insn->length;  in intel_pt_walk_insn()
1036 			err = intel_pt_push(&decoder->stack, decoder->ip +  in intel_pt_walk_insn()
1042 		decoder->ret_addr = intel_pt_pop(&decoder->stack);  in intel_pt_walk_insn()
1046 		int cnt = decoder->no_progress++;  in intel_pt_walk_insn()
1048 		decoder->state.from_ip = decoder->ip;  in intel_pt_walk_insn()
1049 		decoder->ip += intel_pt_insn->length +  in intel_pt_walk_insn()
1051 		decoder->state.to_ip = decoder->ip;  in intel_pt_walk_insn()
1063 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1064 				decoder->stuck_ip_prd = 1;  in intel_pt_walk_insn()
1065 				decoder->stuck_ip_cnt = 1;  in intel_pt_walk_insn()
1067 				   decoder->state.to_ip == decoder->stuck_ip) {  in intel_pt_walk_insn()
1069 						decoder->state.to_ip);  in intel_pt_walk_insn()
1070 				decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_insn()
1073 			} else if (!--decoder->stuck_ip_cnt) {  in intel_pt_walk_insn()
1074 				decoder->stuck_ip_prd += 1;  in intel_pt_walk_insn()
1075 				decoder->stuck_ip_cnt = decoder->stuck_ip_prd;  in intel_pt_walk_insn()
1076 				decoder->stuck_ip = decoder->state.to_ip;  in intel_pt_walk_insn()
1082 	decoder->no_progress = 0;  in intel_pt_walk_insn()
1084 	decoder->state.insn_op = intel_pt_insn->op;  in intel_pt_walk_insn()
1085 	decoder->state.insn_len = intel_pt_insn->length;  in intel_pt_walk_insn()
1086 	memcpy(decoder->state.insn, intel_pt_insn->buf,  in intel_pt_walk_insn()
1089 	if (decoder->tx_flags & INTEL_PT_IN_TX)  in intel_pt_walk_insn()
1090 		decoder->state.flags |= INTEL_PT_IN_TX;  in intel_pt_walk_insn()
1095 static bool intel_pt_fup_event(struct intel_pt_decoder *decoder)  in intel_pt_fup_event()  argument
1099 	if (decoder->set_fup_tx_flags) {  in intel_pt_fup_event()
1100 		decoder->set_fup_tx_flags = false;  in intel_pt_fup_event()
1101 		decoder->tx_flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1102 		decoder->state.type = INTEL_PT_TRANSACTION;  in intel_pt_fup_event()
1103 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1104 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1105 		decoder->state.flags = decoder->fup_tx_flags;  in intel_pt_fup_event()
1108 	if (decoder->set_fup_ptw) {  in intel_pt_fup_event()
1109 		decoder->set_fup_ptw = false;  in intel_pt_fup_event()
1110 		decoder->state.type = INTEL_PT_PTW;  in intel_pt_fup_event()
1111 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1112 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1113 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1114 		decoder->state.ptw_payload = decoder->fup_ptw_payload;  in intel_pt_fup_event()
1117 	if (decoder->set_fup_mwait) {  in intel_pt_fup_event()
1118 		decoder->set_fup_mwait = false;  in intel_pt_fup_event()
1119 		decoder->state.type = INTEL_PT_MWAIT_OP;  in intel_pt_fup_event()
1120 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1121 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1122 		decoder->state.mwait_payload = decoder->fup_mwait_payload;  in intel_pt_fup_event()
1125 	if (decoder->set_fup_pwre) {  in intel_pt_fup_event()
1126 		decoder->set_fup_pwre = false;  in intel_pt_fup_event()
1127 		decoder->state.type |= INTEL_PT_PWR_ENTRY;  in intel_pt_fup_event()
1128 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1129 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1130 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1131 		decoder->state.pwre_payload = decoder->fup_pwre_payload;  in intel_pt_fup_event()
1134 	if (decoder->set_fup_exstop) {  in intel_pt_fup_event()
1135 		decoder->set_fup_exstop = false;  in intel_pt_fup_event()
1136 		decoder->state.type |= INTEL_PT_EX_STOP;  in intel_pt_fup_event()
1137 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1138 		decoder->state.flags |= INTEL_PT_FUP_IP;  in intel_pt_fup_event()
1139 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1140 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1143 	if (decoder->set_fup_bep) {  in intel_pt_fup_event()
1144 		decoder->set_fup_bep = false;  in intel_pt_fup_event()
1145 		decoder->state.type |= INTEL_PT_BLK_ITEMS;  in intel_pt_fup_event()
1146 		decoder->state.type &= ~INTEL_PT_BRANCH;  in intel_pt_fup_event()
1147 		decoder->state.from_ip = decoder->ip;  in intel_pt_fup_event()
1148 		decoder->state.to_ip = 0;  in intel_pt_fup_event()
1154 static inline bool intel_pt_fup_with_nlip(struct intel_pt_decoder *decoder,  in intel_pt_fup_with_nlip()  argument
1158 	return decoder->flags & INTEL_PT_FUP_WITH_NLIP && !err &&  in intel_pt_fup_with_nlip()
1160 	       ip == decoder->ip + intel_pt_insn->length;  in intel_pt_fup_with_nlip()
1163 static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup()  argument
1169 	ip = decoder->last_ip;  in intel_pt_walk_fup()
1172 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);  in intel_pt_walk_fup()
1176 		    intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) {  in intel_pt_walk_fup()
1177 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_fup()
1178 			if (intel_pt_fup_event(decoder))  in intel_pt_walk_fup()
1182 		decoder->set_fup_tx_flags = false;  in intel_pt_walk_fup()
1188 					decoder->ip);  in intel_pt_walk_fup()
1189 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1195 					decoder->ip);  in intel_pt_walk_fup()
1196 			decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_fup()
1200 		intel_pt_bug(decoder);  in intel_pt_walk_fup()
1204 static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_tip()  argument
1209 	err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tip()
1211 	    decoder->pgd_ip &&  in intel_pt_walk_tip()
1212 	    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1213 	    (decoder->state.type & INTEL_PT_BRANCH) &&  in intel_pt_walk_tip()
1214 	    decoder->pgd_ip(decoder->state.to_ip, decoder->data)) {  in intel_pt_walk_tip()
1216 		decoder->no_progress = 0;  in intel_pt_walk_tip()
1217 		decoder->pge = false;  in intel_pt_walk_tip()
1218 		decoder->continuous_period = false;  in intel_pt_walk_tip()
1219 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1220 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1229 		if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {  in intel_pt_walk_tip()
1230 			decoder->pge = false;  in intel_pt_walk_tip()
1231 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1232 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1233 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1234 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1235 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1237 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1238 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1240 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1242 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1243 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1244 			if (decoder->packet.count == 0) {  in intel_pt_walk_tip()
1245 				decoder->state.to_ip = 0;  in intel_pt_walk_tip()
1247 				decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tip()
1248 				decoder->ip = decoder->last_ip;  in intel_pt_walk_tip()
1255 		uint64_t to_ip = decoder->ip + intel_pt_insn.length +  in intel_pt_walk_tip()
1258 		if (decoder->pgd_ip &&  in intel_pt_walk_tip()
1259 		    decoder->pkt_state == INTEL_PT_STATE_TIP_PGD &&  in intel_pt_walk_tip()
1260 		    decoder->pgd_ip(to_ip, decoder->data)) {  in intel_pt_walk_tip()
1262 			decoder->pge = false;  in intel_pt_walk_tip()
1263 			decoder->continuous_period = false;  in intel_pt_walk_tip()
1264 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tip()
1265 			decoder->ip = to_ip;  in intel_pt_walk_tip()
1266 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tip()
1267 			decoder->state.to_ip = to_ip;  in intel_pt_walk_tip()
1268 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_tip()
1272 				decoder->ip);  in intel_pt_walk_tip()
1273 		decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_walk_tip()
1277 	return intel_pt_bug(decoder);  in intel_pt_walk_tip()
1280 static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)  in intel_pt_walk_tnt()  argument
1286 		err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);  in intel_pt_walk_tnt()
1293 			if (!decoder->return_compression) {  in intel_pt_walk_tnt()
1295 						decoder->ip);  in intel_pt_walk_tnt()
1296 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1299 			if (!decoder->ret_addr) {  in intel_pt_walk_tnt()
1301 						decoder->ip);  in intel_pt_walk_tnt()
1302 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1305 			if (!(decoder->tnt.payload & BIT63)) {  in intel_pt_walk_tnt()
1307 						decoder->ip);  in intel_pt_walk_tnt()
1308 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1311 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1312 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1313 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1315 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1316 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1317 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1318 			decoder->ip = decoder->ret_addr;  in intel_pt_walk_tnt()
1319 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1325 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_tnt()
1328 			if (decoder->packet.type != INTEL_PT_TIP ||  in intel_pt_walk_tnt()
1329 			    decoder->packet.count == 0) {  in intel_pt_walk_tnt()
1331 						decoder->ip);  in intel_pt_walk_tnt()
1332 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_tnt()
1333 				decoder->pkt_step = 0;  in intel_pt_walk_tnt()
1336 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_tnt()
1337 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1338 			decoder->state.to_ip = decoder->last_ip;  in intel_pt_walk_tnt()
1339 			decoder->ip = decoder->last_ip;  in intel_pt_walk_tnt()
1344 			decoder->tnt.count -= 1;  in intel_pt_walk_tnt()
1345 			if (decoder->tnt.count)  in intel_pt_walk_tnt()
1346 				decoder->pkt_state = INTEL_PT_STATE_TNT_CONT;  in intel_pt_walk_tnt()
1348 				decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_walk_tnt()
1349 			if (decoder->tnt.payload & BIT63) {  in intel_pt_walk_tnt()
1350 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1351 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1352 				decoder->ip += intel_pt_insn.length +  in intel_pt_walk_tnt()
1354 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_tnt()
1358 			if (decoder->state.type & INTEL_PT_INSTRUCTION) {  in intel_pt_walk_tnt()
1359 				decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1360 				decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_tnt()
1361 				decoder->state.from_ip = decoder->ip;  in intel_pt_walk_tnt()
1362 				decoder->state.to_ip = 0;  in intel_pt_walk_tnt()
1363 				decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1366 			decoder->sample_cyc = false;  in intel_pt_walk_tnt()
1367 			decoder->ip += intel_pt_insn.length;  in intel_pt_walk_tnt()
1368 			if (!decoder->tnt.count) {  in intel_pt_walk_tnt()
1369 				intel_pt_update_sample_time(decoder);  in intel_pt_walk_tnt()
1372 			decoder->tnt.payload <<= 1;  in intel_pt_walk_tnt()
1376 		return intel_pt_bug(decoder);  in intel_pt_walk_tnt()
1380 static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)  in intel_pt_mode_tsx()  argument
1385 	fup_tx_flags = decoder->packet.payload &  in intel_pt_mode_tsx()
1387 	err = intel_pt_get_next_packet(decoder);  in intel_pt_mode_tsx()
1390 	if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_mode_tsx()
1391 		decoder->fup_tx_flags = fup_tx_flags;  in intel_pt_mode_tsx()
1392 		decoder->set_fup_tx_flags = true;  in intel_pt_mode_tsx()
1393 		if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))  in intel_pt_mode_tsx()
1397 				decoder->pos);  in intel_pt_mode_tsx()
1398 		intel_pt_update_in_tx(decoder);  in intel_pt_mode_tsx()
1418 static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_tsc_timestamp()  argument
1422 	decoder->have_tma = false;  in intel_pt_calc_tsc_timestamp()
1424 	if (decoder->ref_timestamp) {  in intel_pt_calc_tsc_timestamp()
1425 		timestamp = intel_pt_8b_tsc(decoder->packet.payload,  in intel_pt_calc_tsc_timestamp()
1426 					    decoder->ref_timestamp);  in intel_pt_calc_tsc_timestamp()
1427 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1428 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1429 		decoder->ref_timestamp = 0;  in intel_pt_calc_tsc_timestamp()
1430 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1431 	} else if (decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1432 		timestamp = decoder->packet.payload |  in intel_pt_calc_tsc_timestamp()
1433 			    (decoder->timestamp & (0xffULL << 56));  in intel_pt_calc_tsc_timestamp()
1434 		decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1435 		if (timestamp < decoder->timestamp &&  in intel_pt_calc_tsc_timestamp()
1436 		    decoder->timestamp - timestamp < decoder->tsc_slip) {  in intel_pt_calc_tsc_timestamp()
1439 			timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1441 		if (timestamp < decoder->timestamp) {  in intel_pt_calc_tsc_timestamp()
1444 			decoder->tsc_timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1446 		decoder->timestamp = timestamp;  in intel_pt_calc_tsc_timestamp()
1447 		decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1450 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_tsc_timestamp()
1451 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_tsc_timestamp()
1452 		decoder->cycle_cnt = 0;  in intel_pt_calc_tsc_timestamp()
1453 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_tsc_timestamp()
1454 		intel_pt_calc_cyc_to_tsc(decoder, false);  in intel_pt_calc_tsc_timestamp()
1457 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_tsc_timestamp()
1460 static int intel_pt_overflow(struct intel_pt_decoder *decoder)  in intel_pt_overflow()  argument
1463 	intel_pt_clear_tx_flags(decoder);  in intel_pt_overflow()
1464 	decoder->timestamp_insn_cnt = 0;  in intel_pt_overflow()
1465 	decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;  in intel_pt_overflow()
1466 	decoder->overflow = true;  in intel_pt_overflow()
1470 static inline void intel_pt_mtc_cyc_cnt_pge(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_pge()  argument
1472 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_pge()
1475 	decoder->cyc_cnt_timestamp = decoder->timestamp;  in intel_pt_mtc_cyc_cnt_pge()
1476 	decoder->base_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_pge()
1479 static inline void intel_pt_mtc_cyc_cnt_cbr(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_cbr()  argument
1481 	decoder->tsc_to_cyc = decoder->cbr / decoder->max_non_turbo_ratio_fp;  in intel_pt_mtc_cyc_cnt_cbr()
1483 	if (decoder->pge)  in intel_pt_mtc_cyc_cnt_cbr()
1484 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_mtc_cyc_cnt_cbr()
1487 static inline void intel_pt_mtc_cyc_cnt_upd(struct intel_pt_decoder *decoder)  in intel_pt_mtc_cyc_cnt_upd()  argument
1491 	if (decoder->have_cyc)  in intel_pt_mtc_cyc_cnt_upd()
1494 	decoder->sample_cyc = true;  in intel_pt_mtc_cyc_cnt_upd()
1496 	if (!decoder->pge || decoder->timestamp <= decoder->cyc_cnt_timestamp)  in intel_pt_mtc_cyc_cnt_upd()
1499 	tsc_delta = decoder->timestamp - decoder->cyc_cnt_timestamp;  in intel_pt_mtc_cyc_cnt_upd()
1500 	tot_cyc_cnt = tsc_delta * decoder->tsc_to_cyc + decoder->base_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1502 	if (tot_cyc_cnt > decoder->tot_cyc_cnt)  in intel_pt_mtc_cyc_cnt_upd()
1503 		decoder->tot_cyc_cnt = tot_cyc_cnt;  in intel_pt_mtc_cyc_cnt_upd()
1506 static void intel_pt_calc_tma(struct intel_pt_decoder *decoder)  in intel_pt_calc_tma()  argument
1508 	uint32_t ctc = decoder->packet.payload;  in intel_pt_calc_tma()
1509 	uint32_t fc = decoder->packet.count;  in intel_pt_calc_tma()
1510 	uint32_t ctc_rem = ctc & decoder->ctc_rem_mask;  in intel_pt_calc_tma()
1512 	if (!decoder->tsc_ctc_ratio_d)  in intel_pt_calc_tma()
1515 	if (decoder->pge && !decoder->in_psb)  in intel_pt_calc_tma()
1516 		intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_calc_tma()
1518 		intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_tma()
1520 	decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff;  in intel_pt_calc_tma()
1521 	decoder->ctc_timestamp = decoder->tsc_timestamp - fc;  in intel_pt_calc_tma()
1522 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_tma()
1523 		decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult;  in intel_pt_calc_tma()
1525 		decoder->ctc_timestamp -= multdiv(ctc_rem,  in intel_pt_calc_tma()
1526 						  decoder->tsc_ctc_ratio_n,  in intel_pt_calc_tma()
1527 						  decoder->tsc_ctc_ratio_d);  in intel_pt_calc_tma()
1529 	decoder->ctc_delta = 0;  in intel_pt_calc_tma()
1530 	decoder->have_tma = true;  in intel_pt_calc_tma()
1531 	decoder->fixup_last_mtc = true;  in intel_pt_calc_tma()
1533 		     decoder->ctc_timestamp, decoder->last_mtc, ctc_rem);  in intel_pt_calc_tma()
1536 static void intel_pt_calc_mtc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_mtc_timestamp()  argument
1541 	if (!decoder->have_tma)  in intel_pt_calc_mtc_timestamp()
1544 	mtc = decoder->packet.payload;  in intel_pt_calc_mtc_timestamp()
1546 	if (decoder->mtc_shift > 8 && decoder->fixup_last_mtc) {  in intel_pt_calc_mtc_timestamp()
1547 		decoder->fixup_last_mtc = false;  in intel_pt_calc_mtc_timestamp()
1548 		intel_pt_fixup_last_mtc(mtc, decoder->mtc_shift,  in intel_pt_calc_mtc_timestamp()
1549 					&decoder->last_mtc);  in intel_pt_calc_mtc_timestamp()
1552 	if (mtc > decoder->last_mtc)  in intel_pt_calc_mtc_timestamp()
1553 		mtc_delta = mtc - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1555 		mtc_delta = mtc + 256 - decoder->last_mtc;  in intel_pt_calc_mtc_timestamp()
1557 	decoder->ctc_delta += mtc_delta << decoder->mtc_shift;  in intel_pt_calc_mtc_timestamp()
1559 	if (decoder->tsc_ctc_mult) {  in intel_pt_calc_mtc_timestamp()
1560 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1561 			    decoder->ctc_delta * decoder->tsc_ctc_mult;  in intel_pt_calc_mtc_timestamp()
1563 		timestamp = decoder->ctc_timestamp +  in intel_pt_calc_mtc_timestamp()
1564 			    multdiv(decoder->ctc_delta,  in intel_pt_calc_mtc_timestamp()
1565 				    decoder->tsc_ctc_ratio_n,  in intel_pt_calc_mtc_timestamp()
1566 				    decoder->tsc_ctc_ratio_d);  in intel_pt_calc_mtc_timestamp()
1569 	if (timestamp < decoder->timestamp)  in intel_pt_calc_mtc_timestamp()
1571 			     timestamp, decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1573 		decoder->timestamp = timestamp;  in intel_pt_calc_mtc_timestamp()
1575 	intel_pt_mtc_cyc_cnt_upd(decoder);  in intel_pt_calc_mtc_timestamp()
1577 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1578 	decoder->last_mtc = mtc;  in intel_pt_calc_mtc_timestamp()
1580 	if (decoder->last_packet_type == INTEL_PT_CYC) {  in intel_pt_calc_mtc_timestamp()
1581 		decoder->cyc_ref_timestamp = decoder->timestamp;  in intel_pt_calc_mtc_timestamp()
1582 		decoder->cycle_cnt = 0;  in intel_pt_calc_mtc_timestamp()
1583 		decoder->have_calc_cyc_to_tsc = false;  in intel_pt_calc_mtc_timestamp()
1584 		intel_pt_calc_cyc_to_tsc(decoder, true);  in intel_pt_calc_mtc_timestamp()
1587 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_mtc_timestamp()
1590 static void intel_pt_calc_cbr(struct intel_pt_decoder *decoder)  in intel_pt_calc_cbr()  argument
1592 	unsigned int cbr = decoder->packet.payload & 0xff;  in intel_pt_calc_cbr()
1594 	decoder->cbr_payload = decoder->packet.payload;  in intel_pt_calc_cbr()
1596 	if (decoder->cbr == cbr)  in intel_pt_calc_cbr()
1599 	decoder->cbr = cbr;  in intel_pt_calc_cbr()
1600 	decoder->cbr_cyc_to_tsc = decoder->max_non_turbo_ratio_fp / cbr;  in intel_pt_calc_cbr()
1602 	intel_pt_mtc_cyc_cnt_cbr(decoder);  in intel_pt_calc_cbr()
1605 static void intel_pt_calc_cyc_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_calc_cyc_timestamp()  argument
1607 	uint64_t timestamp = decoder->cyc_ref_timestamp;  in intel_pt_calc_cyc_timestamp()
1609 	decoder->have_cyc = true;  in intel_pt_calc_cyc_timestamp()
1611 	decoder->cycle_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
1612 	if (decoder->pge)  in intel_pt_calc_cyc_timestamp()
1613 		decoder->tot_cyc_cnt += decoder->packet.payload;  in intel_pt_calc_cyc_timestamp()
1614 	decoder->sample_cyc = true;  in intel_pt_calc_cyc_timestamp()
1616 	if (!decoder->cyc_ref_timestamp)  in intel_pt_calc_cyc_timestamp()
1619 	if (decoder->have_calc_cyc_to_tsc)  in intel_pt_calc_cyc_timestamp()
1620 		timestamp += decoder->cycle_cnt * decoder->calc_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1621 	else if (decoder->cbr)  in intel_pt_calc_cyc_timestamp()
1622 		timestamp += decoder->cycle_cnt * decoder->cbr_cyc_to_tsc;  in intel_pt_calc_cyc_timestamp()
1626 	if (timestamp < decoder->timestamp)  in intel_pt_calc_cyc_timestamp()
1628 			     timestamp, decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
1630 		decoder->timestamp = timestamp;  in intel_pt_calc_cyc_timestamp()
1632 	decoder->timestamp_insn_cnt = 0;  in intel_pt_calc_cyc_timestamp()
1634 	intel_pt_log_to("Setting timestamp", decoder->timestamp);  in intel_pt_calc_cyc_timestamp()
1637 static void intel_pt_bbp(struct intel_pt_decoder *decoder)  in intel_pt_bbp()  argument
1639 	if (decoder->prev_pkt_ctx == INTEL_PT_NO_CTX) {  in intel_pt_bbp()
1640 		memset(decoder->state.items.mask, 0, sizeof(decoder->state.items.mask));  in intel_pt_bbp()
1641 		decoder->state.items.is_32_bit = false;  in intel_pt_bbp()
1643 	decoder->blk_type = decoder->packet.payload;  in intel_pt_bbp()
1644 	decoder->blk_type_pos = intel_pt_blk_type_pos(decoder->blk_type);  in intel_pt_bbp()
1645 	if (decoder->blk_type == INTEL_PT_GP_REGS)  in intel_pt_bbp()
1646 		decoder->state.items.is_32_bit = decoder->packet.count;  in intel_pt_bbp()
1647 	if (decoder->blk_type_pos < 0) {  in intel_pt_bbp()
1649 			     decoder->blk_type);  in intel_pt_bbp()
1650 	} else if (decoder->state.items.mask[decoder->blk_type_pos]) {  in intel_pt_bbp()
1652 			     decoder->blk_type);  in intel_pt_bbp()
1656 static void intel_pt_bip(struct intel_pt_decoder *decoder)  in intel_pt_bip()  argument
1658 	uint32_t id = decoder->packet.count;  in intel_pt_bip()
1660 	int pos = decoder->blk_type_pos;  in intel_pt_bip()
1664 			     id, decoder->blk_type);  in intel_pt_bip()
1668 	if (decoder->state.items.mask[pos] & bit) {  in intel_pt_bip()
1670 			     id, decoder->blk_type);  in intel_pt_bip()
1673 	decoder->state.items.mask[pos] |= bit;  in intel_pt_bip()
1674 	decoder->state.items.val[pos][id] = decoder->packet.payload;  in intel_pt_bip()
1678 static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)  in intel_pt_walk_psbend()  argument
1682 	decoder->in_psb = true;  in intel_pt_walk_psbend()
1685 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psbend()
1689 		switch (decoder->packet.type) {  in intel_pt_walk_psbend()
1712 			decoder->have_tma = false;  in intel_pt_walk_psbend()
1718 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psbend()
1722 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psbend()
1726 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psbend()
1730 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psbend()
1734 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psbend()
1738 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_psbend()
1742 			decoder->pge = true;  in intel_pt_walk_psbend()
1743 			if (decoder->packet.count) {  in intel_pt_walk_psbend()
1744 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_psbend()
1745 				if (decoder->hop) {  in intel_pt_walk_psbend()
1747 					decoder->ip = decoder->last_ip;  in intel_pt_walk_psbend()
1748 					decoder->hop_psb_fup = true;  in intel_pt_walk_psbend()
1754 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psbend()
1758 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psbend()
1759 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_psbend()
1760 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_psbend()
1772 	decoder->in_psb = false;  in intel_pt_walk_psbend()
1777 static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)  in intel_pt_walk_fup_tip()  argument
1781 	if (decoder->tx_flags & INTEL_PT_ABORT_TX) {  in intel_pt_walk_fup_tip()
1782 		decoder->tx_flags = 0;  in intel_pt_walk_fup_tip()
1783 		decoder->state.flags &= ~INTEL_PT_IN_TX;  in intel_pt_walk_fup_tip()
1784 		decoder->state.flags |= INTEL_PT_ABORT_TX;  in intel_pt_walk_fup_tip()
1786 		decoder->state.flags |= INTEL_PT_ASYNC;  in intel_pt_walk_fup_tip()
1790 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_fup_tip()
1794 		switch (decoder->packet.type) {  in intel_pt_walk_fup_tip()
1816 			decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_fup_tip()
1817 			decoder->pkt_step = 0;  in intel_pt_walk_fup_tip()
1821 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_fup_tip()
1825 			return intel_pt_overflow(decoder);  in intel_pt_walk_fup_tip()
1828 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1829 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1830 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1832 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1833 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1835 			decoder->pge = false;  in intel_pt_walk_fup_tip()
1836 			decoder->continuous_period = false;  in intel_pt_walk_fup_tip()
1837 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_fup_tip()
1841 			decoder->pge = true;  in intel_pt_walk_fup_tip()
1843 				     decoder->ip);  in intel_pt_walk_fup_tip()
1844 			decoder->state.from_ip = 0;  in intel_pt_walk_fup_tip()
1845 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1846 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1848 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1849 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1851 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_fup_tip()
1852 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_fup_tip()
1856 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1857 			if (decoder->packet.count == 0) {  in intel_pt_walk_fup_tip()
1858 				decoder->state.to_ip = 0;  in intel_pt_walk_fup_tip()
1860 				intel_pt_set_ip(decoder);  in intel_pt_walk_fup_tip()
1861 				decoder->state.to_ip = decoder->ip;  in intel_pt_walk_fup_tip()
1866 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_fup_tip()
1870 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_fup_tip()
1871 			if (decoder->period_type == INTEL_PT_PERIOD_MTC)  in intel_pt_walk_fup_tip()
1872 				decoder->state.type |= INTEL_PT_INSTRUCTION;  in intel_pt_walk_fup_tip()
1876 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_fup_tip()
1880 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_fup_tip()
1889 			return intel_pt_bug(decoder);  in intel_pt_walk_fup_tip()
1894 static int intel_pt_resample(struct intel_pt_decoder *decoder)  in intel_pt_resample()  argument
1896 	decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_resample()
1897 	decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_resample()
1898 	decoder->state.from_ip = decoder->ip;  in intel_pt_resample()
1899 	decoder->state.to_ip = 0;  in intel_pt_resample()
1908 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder);
1911 static int intel_pt_hop_trace(struct intel_pt_decoder *decoder, bool *no_tip, int *err)  in intel_pt_hop_trace()  argument
1914 	if (decoder->leap && !decoder->in_psb && decoder->packet.type != INTEL_PT_PSB) {  in intel_pt_hop_trace()
1915 		*err = intel_pt_scan_for_psb(decoder);  in intel_pt_hop_trace()
1920 	switch (decoder->packet.type) {  in intel_pt_hop_trace()
1925 		if (!decoder->packet.count)  in intel_pt_hop_trace()
1927 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
1928 		decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_hop_trace()
1929 		decoder->state.from_ip = 0;  in intel_pt_hop_trace()
1930 		decoder->state.to_ip = decoder->ip;  in intel_pt_hop_trace()
1934 		if (!decoder->packet.count)  in intel_pt_hop_trace()
1936 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
1937 		decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
1938 		decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
1939 		decoder->state.to_ip = 0;  in intel_pt_hop_trace()
1943 		if (!decoder->packet.count)  in intel_pt_hop_trace()
1945 		intel_pt_set_ip(decoder);  in intel_pt_hop_trace()
1946 		if (intel_pt_fup_event(decoder))  in intel_pt_hop_trace()
1948 		if (!decoder->branch_enable)  in intel_pt_hop_trace()
1951 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
1952 			decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
1953 			decoder->state.to_ip = 0;  in intel_pt_hop_trace()
1956 		*err = intel_pt_walk_fup_tip(decoder);  in intel_pt_hop_trace()
1958 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_hop_trace()
1962 		decoder->last_ip = 0;  in intel_pt_hop_trace()
1963 		decoder->have_last_ip = true;  in intel_pt_hop_trace()
1964 		decoder->hop_psb_fup = false;  in intel_pt_hop_trace()
1965 		*err = intel_pt_walk_psbend(decoder);  in intel_pt_hop_trace()
1970 		if (decoder->hop_psb_fup) {  in intel_pt_hop_trace()
1971 			decoder->hop_psb_fup = false;  in intel_pt_hop_trace()
1972 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_hop_trace()
1973 			decoder->state.from_ip = decoder->ip;  in intel_pt_hop_trace()
1974 			decoder->state.to_ip = 0;  in intel_pt_hop_trace()
1977 		if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_hop_trace()
1978 			decoder->state.type = 0;  in intel_pt_hop_trace()
2015 static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)  in intel_pt_walk_trace()  argument
2021 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
2025 		if (decoder->hop) {  in intel_pt_walk_trace()
2026 			switch (intel_pt_hop_trace(decoder, &no_tip, &err)) {  in intel_pt_walk_trace()
2038 		switch (decoder->packet.type) {  in intel_pt_walk_trace()
2040 			if (!decoder->packet.count)  in intel_pt_walk_trace()
2042 			decoder->tnt = decoder->packet;  in intel_pt_walk_trace()
2043 			decoder->pkt_state = INTEL_PT_STATE_TNT;  in intel_pt_walk_trace()
2044 			err = intel_pt_walk_tnt(decoder);  in intel_pt_walk_trace()
2050 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
2051 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
2052 			decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;  in intel_pt_walk_trace()
2053 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
2056 			decoder->pge = true;  in intel_pt_walk_trace()
2057 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_trace()
2058 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
2060 						decoder->pos);  in intel_pt_walk_trace()
2063 			intel_pt_set_ip(decoder);  in intel_pt_walk_trace()
2064 			decoder->state.from_ip = 0;  in intel_pt_walk_trace()
2065 			decoder->state.to_ip = decoder->ip;  in intel_pt_walk_trace()
2066 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_trace()
2071 			if (decoder->hop)  in intel_pt_walk_trace()
2072 				decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_walk_trace()
2077 			return intel_pt_overflow(decoder);  in intel_pt_walk_trace()
2080 			if (decoder->packet.count != 0)  in intel_pt_walk_trace()
2081 				intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
2082 			decoder->pkt_state = INTEL_PT_STATE_TIP;  in intel_pt_walk_trace()
2083 			return intel_pt_walk_tip(decoder);  in intel_pt_walk_trace()
2086 			if (decoder->packet.count == 0) {  in intel_pt_walk_trace()
2088 						decoder->pos);  in intel_pt_walk_trace()
2092 			intel_pt_set_last_ip(decoder);  in intel_pt_walk_trace()
2093 			if (!decoder->branch_enable) {  in intel_pt_walk_trace()
2094 				decoder->ip = decoder->last_ip;  in intel_pt_walk_trace()
2095 				if (intel_pt_fup_event(decoder))  in intel_pt_walk_trace()
2100 			if (decoder->set_fup_mwait)  in intel_pt_walk_trace()
2103 				decoder->pkt_state = INTEL_PT_STATE_FUP_NO_TIP;  in intel_pt_walk_trace()
2105 				decoder->pkt_state = INTEL_PT_STATE_FUP;  in intel_pt_walk_trace()
2106 			err = intel_pt_walk_fup(decoder);  in intel_pt_walk_trace()
2113 			return intel_pt_walk_fup_tip(decoder);  in intel_pt_walk_trace()
2116 			decoder->pge = false;  in intel_pt_walk_trace()
2117 			decoder->continuous_period = false;  in intel_pt_walk_trace()
2118 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_trace()
2119 			decoder->have_tma = false;  in intel_pt_walk_trace()
2123 			decoder->last_ip = 0;  in intel_pt_walk_trace()
2124 			decoder->have_last_ip = true;  in intel_pt_walk_trace()
2125 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_trace()
2126 			err = intel_pt_walk_psbend(decoder);  in intel_pt_walk_trace()
2136 			if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
2137 				decoder->state.type = 0;  in intel_pt_walk_trace()
2143 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_trace()
2147 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_trace()
2148 			if (decoder->period_type != INTEL_PT_PERIOD_MTC)  in intel_pt_walk_trace()
2154 			if (!decoder->mtc_insn)  in intel_pt_walk_trace()
2156 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
2158 			if (!decoder->timestamp)  in intel_pt_walk_trace()
2160 			decoder->state.type = INTEL_PT_INSTRUCTION;  in intel_pt_walk_trace()
2161 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2162 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2163 			decoder->mtc_insn = false;  in intel_pt_walk_trace()
2167 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_trace()
2171 			intel_pt_calc_tma(decoder);  in intel_pt_walk_trace()
2175 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_trace()
2179 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_trace()
2180 			if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_walk_trace()
2181 				decoder->state.type = 0;  in intel_pt_walk_trace()
2187 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_trace()
2192 			if (!decoder->pge || decoder->in_psb) {  in intel_pt_walk_trace()
2193 				intel_pt_update_in_tx(decoder);  in intel_pt_walk_trace()
2196 			err = intel_pt_mode_tsx(decoder, &no_tip);  in intel_pt_walk_trace()
2202 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
2211 			decoder->fup_ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
2212 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
2215 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
2216 				decoder->set_fup_ptw = true;  in intel_pt_walk_trace()
2220 						decoder->pos);  in intel_pt_walk_trace()
2225 			decoder->state.type = INTEL_PT_PTW;  in intel_pt_walk_trace()
2226 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2227 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2228 			decoder->state.ptw_payload = decoder->packet.payload;  in intel_pt_walk_trace()
2232 			decoder->fup_mwait_payload = decoder->packet.payload;  in intel_pt_walk_trace()
2233 			decoder->set_fup_mwait = true;  in intel_pt_walk_trace()
2237 			if (decoder->set_fup_mwait) {  in intel_pt_walk_trace()
2238 				decoder->fup_pwre_payload =  in intel_pt_walk_trace()
2239 							decoder->packet.payload;  in intel_pt_walk_trace()
2240 				decoder->set_fup_pwre = true;  in intel_pt_walk_trace()
2243 			decoder->state.type = INTEL_PT_PWR_ENTRY;  in intel_pt_walk_trace()
2244 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2245 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2246 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
2250 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
2253 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
2254 				decoder->set_fup_exstop = true;  in intel_pt_walk_trace()
2258 						decoder->pos);  in intel_pt_walk_trace()
2263 			decoder->state.type = INTEL_PT_EX_STOP;  in intel_pt_walk_trace()
2264 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2265 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2269 			decoder->state.type = INTEL_PT_PWR_EXIT;  in intel_pt_walk_trace()
2270 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2271 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2272 			decoder->state.pwrx_payload = decoder->packet.payload;  in intel_pt_walk_trace()
2276 			intel_pt_bbp(decoder);  in intel_pt_walk_trace()
2280 			intel_pt_bip(decoder);  in intel_pt_walk_trace()
2284 			decoder->state.type = INTEL_PT_BLK_ITEMS;  in intel_pt_walk_trace()
2285 			decoder->state.from_ip = decoder->ip;  in intel_pt_walk_trace()
2286 			decoder->state.to_ip = 0;  in intel_pt_walk_trace()
2290 			err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_trace()
2293 			if (decoder->packet.type == INTEL_PT_FUP) {  in intel_pt_walk_trace()
2294 				decoder->set_fup_bep = true;  in intel_pt_walk_trace()
2298 						decoder->pos);  in intel_pt_walk_trace()
2303 			return intel_pt_bug(decoder);  in intel_pt_walk_trace()
2308 static inline bool intel_pt_have_ip(struct intel_pt_decoder *decoder)  in intel_pt_have_ip()  argument
2310 	return decoder->packet.count &&  in intel_pt_have_ip()
2311 	       (decoder->have_last_ip || decoder->packet.count == 3 ||  in intel_pt_have_ip()
2312 		decoder->packet.count == 6);  in intel_pt_have_ip()
2316 static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)  in intel_pt_walk_psb()  argument
2320 	decoder->in_psb = true;  in intel_pt_walk_psb()
2323 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_psb()
2327 		switch (decoder->packet.type) {  in intel_pt_walk_psb()
2329 			decoder->continuous_period = false;  in intel_pt_walk_psb()
2349 			decoder->pge = true;  in intel_pt_walk_psb()
2350 			if (intel_pt_have_ip(decoder)) {  in intel_pt_walk_psb()
2351 				uint64_t current_ip = decoder->ip;  in intel_pt_walk_psb()
2353 				intel_pt_set_ip(decoder);  in intel_pt_walk_psb()
2356 							decoder->ip);  in intel_pt_walk_psb()
2361 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_psb()
2365 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_psb()
2369 			intel_pt_calc_tma(decoder);  in intel_pt_walk_psb()
2373 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_psb()
2377 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_psb()
2381 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_psb()
2385 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_psb()
2389 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_psb()
2393 			decoder->pge = false;  in intel_pt_walk_psb()
2394 			decoder->continuous_period = false;  in intel_pt_walk_psb()
2395 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_psb()
2399 			decoder->have_tma = false;  in intel_pt_walk_psb()
2401 			if (decoder->ip)  in intel_pt_walk_psb()
2402 				decoder->pkt_state = INTEL_PT_STATE_ERR4;  in intel_pt_walk_psb()
2404 				decoder->pkt_state = INTEL_PT_STATE_ERR3;  in intel_pt_walk_psb()
2409 			err = intel_pt_bug(decoder);  in intel_pt_walk_psb()
2413 			err = intel_pt_overflow(decoder);  in intel_pt_walk_psb()
2429 	decoder->in_psb = false;  in intel_pt_walk_psb()
2434 static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)  in intel_pt_walk_to_ip()  argument
2439 		err = intel_pt_get_next_packet(decoder);  in intel_pt_walk_to_ip()
2443 		switch (decoder->packet.type) {  in intel_pt_walk_to_ip()
2445 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
2446 			decoder->pge = false;  in intel_pt_walk_to_ip()
2447 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2448 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2449 			if (!decoder->ip)  in intel_pt_walk_to_ip()
2451 			decoder->state.type |= INTEL_PT_TRACE_END;  in intel_pt_walk_to_ip()
2455 			decoder->pge = true;  in intel_pt_walk_to_ip()
2456 			intel_pt_mtc_cyc_cnt_pge(decoder);  in intel_pt_walk_to_ip()
2457 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2458 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2459 			if (!decoder->ip)  in intel_pt_walk_to_ip()
2461 			decoder->state.type |= INTEL_PT_TRACE_BEGIN;  in intel_pt_walk_to_ip()
2465 			decoder->pge = true;  in intel_pt_walk_to_ip()
2466 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2467 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2468 			if (!decoder->ip)  in intel_pt_walk_to_ip()
2473 			if (intel_pt_have_ip(decoder))  in intel_pt_walk_to_ip()
2474 				intel_pt_set_ip(decoder);  in intel_pt_walk_to_ip()
2475 			if (decoder->ip)  in intel_pt_walk_to_ip()
2480 			intel_pt_calc_mtc_timestamp(decoder);  in intel_pt_walk_to_ip()
2484 			intel_pt_calc_tsc_timestamp(decoder);  in intel_pt_walk_to_ip()
2488 			intel_pt_calc_tma(decoder);  in intel_pt_walk_to_ip()
2492 			intel_pt_calc_cyc_timestamp(decoder);  in intel_pt_walk_to_ip()
2496 			intel_pt_calc_cbr(decoder);  in intel_pt_walk_to_ip()
2500 			decoder->cr3 = decoder->packet.payload & (BIT63 - 1);  in intel_pt_walk_to_ip()
2504 			decoder->exec_mode = decoder->packet.payload;  in intel_pt_walk_to_ip()
2508 			intel_pt_update_in_tx(decoder);  in intel_pt_walk_to_ip()
2512 			return intel_pt_overflow(decoder);  in intel_pt_walk_to_ip()
2515 			return intel_pt_bug(decoder);  in intel_pt_walk_to_ip()
2518 			decoder->pge = false;  in intel_pt_walk_to_ip()
2519 			decoder->continuous_period = false;  in intel_pt_walk_to_ip()
2520 			intel_pt_clear_tx_flags(decoder);  in intel_pt_walk_to_ip()
2521 			decoder->have_tma = false;  in intel_pt_walk_to_ip()
2525 			decoder->last_ip = 0;  in intel_pt_walk_to_ip()
2526 			decoder->have_last_ip = true;  in intel_pt_walk_to_ip()
2527 			intel_pt_clear_stack(&decoder->stack);  in intel_pt_walk_to_ip()
2528 			err = intel_pt_walk_psb(decoder);  in intel_pt_walk_to_ip()
2531 			if (decoder->ip) {  in intel_pt_walk_to_ip()
2533 				decoder->state.type = 0;  in intel_pt_walk_to_ip()
2560 static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)  in intel_pt_sync_ip()  argument
2564 	decoder->set_fup_tx_flags = false;  in intel_pt_sync_ip()
2565 	decoder->set_fup_ptw = false;  in intel_pt_sync_ip()
2566 	decoder->set_fup_mwait = false;  in intel_pt_sync_ip()
2567 	decoder->set_fup_pwre = false;  in intel_pt_sync_ip()
2568 	decoder->set_fup_exstop = false;  in intel_pt_sync_ip()
2569 	decoder->set_fup_bep = false;  in intel_pt_sync_ip()
2571 	if (!decoder->branch_enable) {  in intel_pt_sync_ip()
2572 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
2573 		decoder->overflow = false;  in intel_pt_sync_ip()
2574 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync_ip()
2579 	err = intel_pt_walk_to_ip(decoder);  in intel_pt_sync_ip()
2584 	if (decoder->hop)  in intel_pt_sync_ip()
2585 		decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync_ip()
2587 		decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync_ip()
2588 	decoder->overflow = false;  in intel_pt_sync_ip()
2590 	decoder->state.from_ip = 0;  in intel_pt_sync_ip()
2591 	decoder->state.to_ip = decoder->ip;  in intel_pt_sync_ip()
2592 	intel_pt_log_to("Setting IP", decoder->ip);  in intel_pt_sync_ip()
2597 static int intel_pt_part_psb(struct intel_pt_decoder *decoder)  in intel_pt_part_psb()  argument
2599 	const unsigned char *end = decoder->buf + decoder->len;  in intel_pt_part_psb()
2603 		if (i > decoder->len)  in intel_pt_part_psb()
2611 static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)  in intel_pt_rest_psb()  argument
2616 	if (rest_psb > decoder->len ||  in intel_pt_rest_psb()
2617 	    memcmp(decoder->buf, psb + part_psb, rest_psb))  in intel_pt_rest_psb()
2623 static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,  in intel_pt_get_split_psb()  argument
2628 	decoder->pos += decoder->len;  in intel_pt_get_split_psb()
2629 	decoder->len = 0;  in intel_pt_get_split_psb()
2631 	ret = intel_pt_get_next_data(decoder, false);  in intel_pt_get_split_psb()
2635 	rest_psb = intel_pt_rest_psb(decoder, part_psb);  in intel_pt_get_split_psb()
2639 	decoder->pos -= part_psb;  in intel_pt_get_split_psb()
2640 	decoder->next_buf = decoder->buf + rest_psb;  in intel_pt_get_split_psb()
2641 	decoder->next_len = decoder->len - rest_psb;  in intel_pt_get_split_psb()
2642 	memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);  in intel_pt_get_split_psb()
2643 	decoder->buf = decoder->temp_buf;  in intel_pt_get_split_psb()
2644 	decoder->len = INTEL_PT_PSB_LEN;  in intel_pt_get_split_psb()
2649 static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)  in intel_pt_scan_for_psb()  argument
2656 		if (!decoder->len) {  in intel_pt_scan_for_psb()
2657 			ret = intel_pt_get_next_data(decoder, false);  in intel_pt_scan_for_psb()
2662 		next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,  in intel_pt_scan_for_psb()
2667 			part_psb = intel_pt_part_psb(decoder);  in intel_pt_scan_for_psb()
2669 				ret = intel_pt_get_split_psb(decoder, part_psb);  in intel_pt_scan_for_psb()
2673 				decoder->pos += decoder->len;  in intel_pt_scan_for_psb()
2674 				decoder->len = 0;  in intel_pt_scan_for_psb()
2679 		decoder->pkt_step = next - decoder->buf;  in intel_pt_scan_for_psb()
2680 		return intel_pt_get_next_packet(decoder);  in intel_pt_scan_for_psb()
2684 static int intel_pt_sync(struct intel_pt_decoder *decoder)  in intel_pt_sync()  argument
2688 	decoder->pge = false;  in intel_pt_sync()
2689 	decoder->continuous_period = false;  in intel_pt_sync()
2690 	decoder->have_last_ip = false;  in intel_pt_sync()
2691 	decoder->last_ip = 0;  in intel_pt_sync()
2692 	decoder->ip = 0;  in intel_pt_sync()
2693 	intel_pt_clear_stack(&decoder->stack);  in intel_pt_sync()
2696 	err = intel_pt_scan_for_psb(decoder);  in intel_pt_sync()
2700 	decoder->have_last_ip = true;  in intel_pt_sync()
2701 	decoder->pkt_state = INTEL_PT_STATE_NO_IP;  in intel_pt_sync()
2703 	err = intel_pt_walk_psb(decoder);  in intel_pt_sync()
2707 	if (decoder->ip) {  in intel_pt_sync()
2708 		decoder->state.type = 0; /* Do not have a sample */  in intel_pt_sync()
2713 		if (decoder->hop)  in intel_pt_sync()
2714 			decoder->pkt_state = INTEL_PT_STATE_RESAMPLE;  in intel_pt_sync()
2716 			decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;  in intel_pt_sync()
2717 	} else if (decoder->leap) {  in intel_pt_sync()
2724 		return intel_pt_sync_ip(decoder);  in intel_pt_sync()
2730 static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)  in intel_pt_est_timestamp()  argument
2732 	uint64_t est = decoder->sample_insn_cnt << 1;  in intel_pt_est_timestamp()
2734 	if (!decoder->cbr || !decoder->max_non_turbo_ratio)  in intel_pt_est_timestamp()
2737 	est *= decoder->max_non_turbo_ratio;  in intel_pt_est_timestamp()
2738 	est /= decoder->cbr;  in intel_pt_est_timestamp()
2740 	return decoder->sample_timestamp + est;  in intel_pt_est_timestamp()
2743 const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)  in intel_pt_decode()  argument
2748 		decoder->state.type = INTEL_PT_BRANCH;  in intel_pt_decode()
2749 		decoder->state.flags = 0;  in intel_pt_decode()
2751 		switch (decoder->pkt_state) {  in intel_pt_decode()
2753 			err = intel_pt_sync(decoder);  in intel_pt_decode()
2756 			decoder->have_last_ip = false;  in intel_pt_decode()
2757 			decoder->last_ip = 0;  in intel_pt_decode()
2758 			decoder->ip = 0;  in intel_pt_decode()
2761 			err = intel_pt_sync_ip(decoder);  in intel_pt_decode()
2764 			err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2768 			err = intel_pt_walk_tnt(decoder);  in intel_pt_decode()
2770 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2774 			err = intel_pt_walk_tip(decoder);  in intel_pt_decode()
2777 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
2779 				err = intel_pt_walk_fup_tip(decoder);  in intel_pt_decode()
2782 			err = intel_pt_walk_fup(decoder);  in intel_pt_decode()
2784 				err = intel_pt_walk_trace(decoder);  in intel_pt_decode()
2787 			err = intel_pt_resample(decoder);  in intel_pt_decode()
2790 			err = intel_pt_bug(decoder);  in intel_pt_decode()
2796 		decoder->state.err = intel_pt_ext_err(err);  in intel_pt_decode()
2797 		decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
2798 		intel_pt_update_sample_time(decoder);  in intel_pt_decode()
2799 		decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
2801 		decoder->state.err = 0;  in intel_pt_decode()
2802 		if (decoder->cbr != decoder->cbr_seen) {  in intel_pt_decode()
2803 			decoder->cbr_seen = decoder->cbr;  in intel_pt_decode()
2804 			if (!decoder->state.type) {  in intel_pt_decode()
2805 				decoder->state.from_ip = decoder->ip;  in intel_pt_decode()
2806 				decoder->state.to_ip = 0;  in intel_pt_decode()
2808 			decoder->state.type |= INTEL_PT_CBR_CHG;  in intel_pt_decode()
2809 			decoder->state.cbr_payload = decoder->cbr_payload;  in intel_pt_decode()
2810 			decoder->state.cbr = decoder->cbr;  in intel_pt_decode()
2812 		if (intel_pt_sample_time(decoder->pkt_state)) {  in intel_pt_decode()
2813 			intel_pt_update_sample_time(decoder);  in intel_pt_decode()
2814 			if (decoder->sample_cyc)  in intel_pt_decode()
2815 				decoder->sample_tot_cyc_cnt = decoder->tot_cyc_cnt;  in intel_pt_decode()
2819 	decoder->state.timestamp = decoder->sample_timestamp;  in intel_pt_decode()
2820 	decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);  in intel_pt_decode()
2821 	decoder->state.cr3 = decoder->cr3;  in intel_pt_decode()
2822 	decoder->state.tot_insn_cnt = decoder->tot_insn_cnt;  in intel_pt_decode()
2823 	decoder->state.tot_cyc_cnt = decoder->sample_tot_cyc_cnt;  in intel_pt_decode()
2825 	return &decoder->state;  in intel_pt_decode()
3197 int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp)  in intel_pt_fast_forward()  argument
3207 	err = decoder->lookahead(decoder->data, intel_pt_ff_cb, &d);  in intel_pt_fast_forward()
3214 			decoder->pos += decoder->len;  in intel_pt_fast_forward()
3215 			decoder->len = 0;  in intel_pt_fast_forward()
3216 			err = intel_pt_get_next_data(decoder, true);  in intel_pt_fast_forward()
3220 		} while (decoder->buf_timestamp != d.buf_timestamp);  in intel_pt_fast_forward()
3223 	if (!decoder->buf)  in intel_pt_fast_forward()
3226 	buf = (unsigned char *)decoder->buf;  in intel_pt_fast_forward()
3227 	len = decoder->len;  in intel_pt_fast_forward()
3242 		tsc = intel_pt_8b_tsc(tsc, decoder->buf_timestamp);  in intel_pt_fast_forward()
3251 			decoder->pos += decoder->len - len;  in intel_pt_fast_forward()
3252 			decoder->buf = buf;  in intel_pt_fast_forward()
3253 			decoder->len = len;  in intel_pt_fast_forward()
3254 			intel_pt_reposition(decoder);  in intel_pt_fast_forward()