282 				 struct perf_sample *sample)  in intel_pt_dump_sample()  argument
288 	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);  in intel_pt_dump_sample()
1206 				   struct perf_sample *sample)  in intel_pt_add_callchain()  argument
1209 							sample->pid,  in intel_pt_add_callchain()
1210 							sample->tid);  in intel_pt_add_callchain()
1212 	thread_stack__sample_late(thread, sample->cpu, pt->chain,  in intel_pt_add_callchain()
1213 				  pt->synth_opts.callchain_sz + 1, sample->ip,  in intel_pt_add_callchain()
1216 	sample->callchain = pt->chain;  in intel_pt_add_callchain()
1244 				  struct perf_sample *sample)  in intel_pt_add_br_stack()  argument
1247 							sample->pid,  in intel_pt_add_br_stack()
1248 							sample->tid);  in intel_pt_add_br_stack()
1250 	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,  in intel_pt_add_br_stack()
1251 				     pt->br_stack_sz, sample->ip,  in intel_pt_add_br_stack()
1254 	sample->branch_stack = pt->br_stack;  in intel_pt_add_br_stack()
1654 				   struct perf_sample *sample)  in intel_pt_prep_a_sample()  argument
1656 	event->sample.header.type = PERF_RECORD_SAMPLE;  in intel_pt_prep_a_sample()
1657 	event->sample.header.size = sizeof(struct perf_event_header);  in intel_pt_prep_a_sample()
1659 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1660 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1665 			sample->pid = ptq->guest_pid;  in intel_pt_prep_a_sample()
1666 			sample->tid = ptq->guest_tid;  in intel_pt_prep_a_sample()
1667 			sample->machine_pid = ptq->guest_machine_pid;  in intel_pt_prep_a_sample()
1668 			sample->vcpu = ptq->vcpu;  in intel_pt_prep_a_sample()
1672 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1673 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1674 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1680 				   struct perf_sample *sample)  in intel_pt_prep_b_sample()  argument
1682 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1685 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1687 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1688 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1689 	sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr);  in intel_pt_prep_b_sample()
1690 	sample->period = 1;  in intel_pt_prep_b_sample()
1691 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1693 	event->sample.header.misc = sample->cpumode;  in intel_pt_prep_b_sample()
1697 				 struct perf_sample *sample, u64 type)  in intel_pt_inject_event()  argument
1699 	event->header.size = perf_event__sample_event_size(sample, type, 0);  in intel_pt_inject_event()
1700 	return perf_event__synthesize_sample(event, type, 0, sample);  in intel_pt_inject_event()
1705 				      struct perf_sample *sample, u64 type)  in intel_pt_opt_inject()  argument
1710 	return intel_pt_inject_event(event, sample, type);  in intel_pt_opt_inject()
1715 					struct perf_sample *sample, u64 type)  in intel_pt_deliver_synth_event()  argument
1719 	ret = intel_pt_opt_inject(pt, event, sample, type);  in intel_pt_deliver_synth_event()
1723 	ret = perf_session__deliver_synth_event(pt->session, event, sample);  in intel_pt_deliver_synth_event()
1734 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_branch_sample()  local
1747 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1749 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1750 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1761 				.from = sample.ip,  in intel_pt_synth_branch_sample()
1762 				.to = sample.addr,  in intel_pt_synth_branch_sample()
1765 		sample.branch_stack = (struct branch_stack *)&dummy_bs;  in intel_pt_synth_branch_sample()
1769 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1770 	if (sample.cyc_cnt) {  in intel_pt_synth_branch_sample()
1771 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1776 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_branch_sample()
1783 				 struct perf_sample *sample)  in intel_pt_prep_sample()  argument
1785 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1790 				     sample->ip, pt->kernel_start);  in intel_pt_prep_sample()
1791 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1797 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1805 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_instruction_sample()  local
1810 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1812 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1813 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1815 		sample.period = 1;  in intel_pt_synth_instruction_sample()
1817 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1820 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1821 	if (sample.cyc_cnt) {  in intel_pt_synth_instruction_sample()
1822 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1829 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_instruction_sample()
1837 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_transaction_sample()  local
1842 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1844 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1845 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1847 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_transaction_sample()
1854 				   struct perf_sample *sample)  in intel_pt_prep_p_sample()  argument
1856 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1862 	if (!sample->ip)  in intel_pt_prep_p_sample()
1863 		sample->flags = 0;  in intel_pt_prep_p_sample()
1870 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_ptwrite_sample()  local
1876 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1878 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1879 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1885 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_ptwrite_sample()
1886 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_ptwrite_sample()
1888 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_ptwrite_sample()
1896 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cbr_sample()  local
1905 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1907 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1908 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1915 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_cbr_sample()
1916 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_cbr_sample()
1918 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_cbr_sample()
1926 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_psb_sample()  local
1932 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_psb_sample()
1934 	sample.id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1935 	sample.stream_id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1936 	sample.flags = 0;  in intel_pt_synth_psb_sample()
1941 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_psb_sample()
1942 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_psb_sample()
1944 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_psb_sample()
1952 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_mwait_sample()  local
1958 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1960 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1961 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1966 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_mwait_sample()
1967 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_mwait_sample()
1969 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_mwait_sample()
1977 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwre_sample()  local
1983 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1985 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1986 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1991 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwre_sample()
1992 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwre_sample()
1994 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwre_sample()
2002 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_exstop_sample()  local
2008 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
2010 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2011 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
2016 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_exstop_sample()
2017 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_exstop_sample()
2019 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_exstop_sample()
2027 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwrx_sample()  local
2033 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
2035 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2036 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
2041 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwrx_sample()
2042 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwrx_sample()
2044 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwrx_sample()
2178 	struct perf_sample sample = { .ip = 0, };  in intel_pt_do_synth_pebs_sample()  local
2183 	u64 regs[8 * sizeof(sample.intr_regs.mask)];  in intel_pt_do_synth_pebs_sample()
2188 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_do_synth_pebs_sample()
2190 	sample.id = id;  in intel_pt_do_synth_pebs_sample()
2191 	sample.stream_id = id;  in intel_pt_do_synth_pebs_sample()
2194 		sample.period = evsel->core.attr.sample_period;  in intel_pt_do_synth_pebs_sample()
2198 		sample.ip = items->ip;  in intel_pt_do_synth_pebs_sample()
2200 		sample.ip = items->rip;  in intel_pt_do_synth_pebs_sample()
2202 		sample.ip = ptq->state->from_ip;  in intel_pt_do_synth_pebs_sample()
2204 	cpumode = intel_pt_cpumode(ptq, sample.ip, 0);  in intel_pt_do_synth_pebs_sample()
2206 	event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;  in intel_pt_do_synth_pebs_sample()
2208 	sample.cpumode = cpumode;  in intel_pt_do_synth_pebs_sample()
2218 			sample.time = tsc_to_perf_time(timestamp, &pt->tc);  in intel_pt_do_synth_pebs_sample()
2224 				     pt->synth_opts.callchain_sz, sample.ip,  in intel_pt_do_synth_pebs_sample()
2226 		sample.callchain = ptq->chain;  in intel_pt_do_synth_pebs_sample()
2235 		sample.intr_regs.abi = items->is_32_bit ?  in intel_pt_do_synth_pebs_sample()
2238 		sample.intr_regs.regs = regs;  in intel_pt_do_synth_pebs_sample()
2240 		pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);  in intel_pt_do_synth_pebs_sample()
2242 		intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);  in intel_pt_do_synth_pebs_sample()
2257 		sample.branch_stack = ptq->last_branch;  in intel_pt_do_synth_pebs_sample()
2261 		sample.addr = items->mem_access_address;  in intel_pt_do_synth_pebs_sample()
2282 				sample.weight = weight & 0xffff;  in intel_pt_do_synth_pebs_sample()
2283 				sample.ins_lat = items->mem_access_latency & 0xffff;  in intel_pt_do_synth_pebs_sample()
2285 				sample.weight = items->mem_access_latency;  in intel_pt_do_synth_pebs_sample()
2287 		if (!sample.weight && items->has_tsx_aux_info) {  in intel_pt_do_synth_pebs_sample()
2289 			sample.weight = (u32)items->tsx_aux_info;  in intel_pt_do_synth_pebs_sample()
2301 		sample.transaction = txn;  in intel_pt_do_synth_pebs_sample()
2304 	return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);  in intel_pt_do_synth_pebs_sample()
2350 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_events_sample()  local
2360 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_events_sample()
2362 	sample.id        = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2363 	sample.stream_id = ptq->pt->evt_id;  in intel_pt_synth_events_sample()
2377 	sample.raw_size = perf_synth__raw_size(raw) +  in intel_pt_synth_events_sample()
2379 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_events_sample()
2381 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_events_sample()
2389 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_iflag_chg_sample()  local
2395 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_iflag_chg_sample()
2397 	sample.id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2398 	sample.stream_id = ptq->pt->iflag_chg_id;  in intel_pt_synth_iflag_chg_sample()
2407 		sample.addr = 0;  in intel_pt_synth_iflag_chg_sample()
2409 	sample.flags = ptq->flags;  in intel_pt_synth_iflag_chg_sample()
2411 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_iflag_chg_sample()
2412 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_iflag_chg_sample()
2414 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_iflag_chg_sample()
3017 					    struct perf_sample *sample)  in intel_pt_sample_set_pid_tid_cpu()  argument
3021 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
3022 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
3044 					    struct perf_sample *sample)  in intel_pt_process_timeless_sample()  argument
3050 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_timeless_sample()
3059 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
3060 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
3065 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)  in intel_pt_lost()  argument
3067 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,  in intel_pt_lost()
3068 				    sample->pid, sample->tid, 0, sample->time,  in intel_pt_lost()
3069 				    sample->machine_pid, sample->vcpu);  in intel_pt_lost()
3146 				   struct perf_sample *sample)  in intel_pt_process_switch()  argument
3150 	struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id);  in intel_pt_process_switch()
3155 	tid = evsel__intval(evsel, sample, "next_pid");  in intel_pt_process_switch()
3156 	cpu = sample->cpu;  in intel_pt_process_switch()
3159 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,  in intel_pt_process_switch()
3162 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_process_switch()
3170 				      struct perf_sample *sample)  in intel_pt_context_switch_in()  argument
3172 	pid_t pid = sample->pid;  in intel_pt_context_switch_in()
3173 	pid_t tid = sample->tid;  in intel_pt_context_switch_in()
3174 	int cpu = sample->cpu;  in intel_pt_context_switch_in()
3209 					 struct perf_sample *sample)  in intel_pt_guest_context_switch()  argument
3213 	struct machine *machine = machines__find(machines, sample->machine_pid);  in intel_pt_guest_context_switch()
3231 	return machine__set_current_tid(machine, sample->vcpu, sample->pid, sample->tid);  in intel_pt_guest_context_switch()
3235 				   struct perf_sample *sample)  in intel_pt_context_switch()  argument
3242 		return intel_pt_guest_context_switch(pt, event, sample);  in intel_pt_context_switch()
3244 	cpu = sample->cpu;  in intel_pt_context_switch()
3248 			return intel_pt_context_switch_in(pt, sample);  in intel_pt_context_switch()
3258 		pid = sample->pid;  in intel_pt_context_switch()
3259 		tid = sample->tid;  in intel_pt_context_switch()
3265 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_context_switch()
3274 					 struct perf_sample *sample)  in intel_pt_process_itrace_start()  argument
3280 		     sample->cpu, event->itrace_start.pid,  in intel_pt_process_itrace_start()
3281 		     event->itrace_start.tid, sample->time,  in intel_pt_process_itrace_start()
3282 		     perf_time_to_tsc(sample->time, &pt->tc));  in intel_pt_process_itrace_start()
3284 	return machine__set_current_tid(pt->machine, sample->cpu,  in intel_pt_process_itrace_start()
3291 					     struct perf_sample *sample)  in intel_pt_process_aux_output_hw_id()  argument
3298 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_aux_output_hw_id()
3299 	evsel = evlist__id2evsel_strict(pt->session->evlist, sample->id);  in intel_pt_process_aux_output_hw_id()
3308 	ptq->pebs[hw_id].id = sample->id;  in intel_pt_process_aux_output_hw_id()
3376 				  struct perf_sample *sample,  in intel_pt_process_event()  argument
3392 	if (sample->time && sample->time != (u64)-1)  in intel_pt_process_event()
3393 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_process_event()
3405 			if (sample->aux_sample.size)  in intel_pt_process_event()
3407 								       sample);  in intel_pt_process_event()
3411 							       sample->time);  in intel_pt_process_event()
3422 		if (pt->synth_opts.add_callchain && !sample->callchain)  in intel_pt_process_event()
3423 			intel_pt_add_callchain(pt, sample);  in intel_pt_process_event()
3424 		if (pt->synth_opts.add_last_branch && !sample->branch_stack)  in intel_pt_process_event()
3425 			intel_pt_add_br_stack(pt, sample);  in intel_pt_process_event()
3431 		err = intel_pt_lost(pt, sample);  in intel_pt_process_event()
3437 		err = intel_pt_process_switch(pt, sample);  in intel_pt_process_event()
3439 		err = intel_pt_process_itrace_start(pt, event, sample);  in intel_pt_process_event()
3441 		err = intel_pt_process_aux_output_hw_id(pt, event, sample);  in intel_pt_process_event()
3444 		err = intel_pt_context_switch(pt, event, sample);  in intel_pt_process_event()
3449 	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {  in intel_pt_process_event()
3451 			     event->header.type, sample->cpu, sample->time, timestamp);  in intel_pt_process_event()
3562 			       struct perf_sample *sample,  in intel_pt_queue_data()  argument
3574 	if (sample->time && sample->time != (u64)-1)  in intel_pt_queue_data()
3575 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_queue_data()
3579 	return auxtrace_queues__add_sample(&pt->queues, session, sample,  in intel_pt_queue_data()
3590 				struct perf_sample *sample __maybe_unused,  in intel_pt_event_synth()