252 				 struct perf_sample *sample)  in intel_pt_dump_sample()  argument
258 	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);  in intel_pt_dump_sample()
1097 				   struct perf_sample *sample)  in intel_pt_add_callchain()  argument
1100 							sample->pid,  in intel_pt_add_callchain()
1101 							sample->tid);  in intel_pt_add_callchain()
1103 	thread_stack__sample_late(thread, sample->cpu, pt->chain,  in intel_pt_add_callchain()
1104 				  pt->synth_opts.callchain_sz + 1, sample->ip,  in intel_pt_add_callchain()
1107 	sample->callchain = pt->chain;  in intel_pt_add_callchain()
1135 				  struct perf_sample *sample)  in intel_pt_add_br_stack()  argument
1138 							sample->pid,  in intel_pt_add_br_stack()
1139 							sample->tid);  in intel_pt_add_br_stack()
1141 	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,  in intel_pt_add_br_stack()
1142 				     pt->br_stack_sz, sample->ip,  in intel_pt_add_br_stack()
1145 	sample->branch_stack = pt->br_stack;  in intel_pt_add_br_stack()
1471 				   struct perf_sample *sample)  in intel_pt_prep_a_sample()  argument
1473 	event->sample.header.type = PERF_RECORD_SAMPLE;  in intel_pt_prep_a_sample()
1474 	event->sample.header.size = sizeof(struct perf_event_header);  in intel_pt_prep_a_sample()
1476 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1477 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1478 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1479 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1480 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1486 				   struct perf_sample *sample)  in intel_pt_prep_b_sample()  argument
1488 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1491 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1493 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1494 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1495 	sample->cpumode = intel_pt_cpumode(ptq, sample->ip, sample->addr);  in intel_pt_prep_b_sample()
1496 	sample->period = 1;  in intel_pt_prep_b_sample()
1497 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1499 	event->sample.header.misc = sample->cpumode;  in intel_pt_prep_b_sample()
1503 				 struct perf_sample *sample, u64 type)  in intel_pt_inject_event()  argument
1505 	event->header.size = perf_event__sample_event_size(sample, type, 0);  in intel_pt_inject_event()
1506 	return perf_event__synthesize_sample(event, type, 0, sample);  in intel_pt_inject_event()
1511 				      struct perf_sample *sample, u64 type)  in intel_pt_opt_inject()  argument
1516 	return intel_pt_inject_event(event, sample, type);  in intel_pt_opt_inject()
1521 					struct perf_sample *sample, u64 type)  in intel_pt_deliver_synth_event()  argument
1525 	ret = intel_pt_opt_inject(pt, event, sample, type);  in intel_pt_deliver_synth_event()
1529 	ret = perf_session__deliver_synth_event(pt->session, event, sample);  in intel_pt_deliver_synth_event()
1540 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_branch_sample()  local
1553 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1555 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1556 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1567 				.from = sample.ip,  in intel_pt_synth_branch_sample()
1568 				.to = sample.addr,  in intel_pt_synth_branch_sample()
1571 		sample.branch_stack = (struct branch_stack *)&dummy_bs;  in intel_pt_synth_branch_sample()
1575 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1576 	if (sample.cyc_cnt) {  in intel_pt_synth_branch_sample()
1577 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1582 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_branch_sample()
1589 				 struct perf_sample *sample)  in intel_pt_prep_sample()  argument
1591 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1596 				     sample->ip, pt->kernel_start);  in intel_pt_prep_sample()
1597 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1603 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1611 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_instruction_sample()  local
1616 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1618 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1619 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1621 		sample.period = 1;  in intel_pt_synth_instruction_sample()
1623 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1626 		sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1627 	if (sample.cyc_cnt) {  in intel_pt_synth_instruction_sample()
1628 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1635 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_instruction_sample()
1643 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_transaction_sample()  local
1648 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1650 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1651 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1653 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_transaction_sample()
1660 				   struct perf_sample *sample)  in intel_pt_prep_p_sample()  argument
1662 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1668 	if (!sample->ip)  in intel_pt_prep_p_sample()
1669 		sample->flags = 0;  in intel_pt_prep_p_sample()
1676 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_ptwrite_sample()  local
1682 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1684 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1685 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1691 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_ptwrite_sample()
1692 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_ptwrite_sample()
1694 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_ptwrite_sample()
1702 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cbr_sample()  local
1711 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1713 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1714 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1721 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_cbr_sample()
1722 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_cbr_sample()
1724 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_cbr_sample()
1732 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_psb_sample()  local
1738 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_psb_sample()
1740 	sample.id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1741 	sample.stream_id = ptq->pt->psb_id;  in intel_pt_synth_psb_sample()
1742 	sample.flags = 0;  in intel_pt_synth_psb_sample()
1747 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_psb_sample()
1748 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_psb_sample()
1750 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_psb_sample()
1758 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_mwait_sample()  local
1764 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1766 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1767 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1772 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_mwait_sample()
1773 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_mwait_sample()
1775 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_mwait_sample()
1783 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwre_sample()  local
1789 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1791 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1792 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1797 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwre_sample()
1798 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwre_sample()
1800 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwre_sample()
1808 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_exstop_sample()  local
1814 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
1816 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1817 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1822 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_exstop_sample()
1823 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_exstop_sample()
1825 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_exstop_sample()
1833 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwrx_sample()  local
1839 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
1841 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1842 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1847 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwrx_sample()
1848 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwrx_sample()
1850 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwrx_sample()
1984 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pebs_sample()  local
1991 	u64 regs[8 * sizeof(sample.intr_regs.mask)];  in intel_pt_synth_pebs_sample()
1996 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_synth_pebs_sample()
1998 	sample.id = id;  in intel_pt_synth_pebs_sample()
1999 	sample.stream_id = id;  in intel_pt_synth_pebs_sample()
2002 		sample.period = evsel->core.attr.sample_period;  in intel_pt_synth_pebs_sample()
2006 		sample.ip = items->ip;  in intel_pt_synth_pebs_sample()
2008 		sample.ip = items->rip;  in intel_pt_synth_pebs_sample()
2010 		sample.ip = ptq->state->from_ip;  in intel_pt_synth_pebs_sample()
2012 	cpumode = intel_pt_cpumode(ptq, sample.ip, 0);  in intel_pt_synth_pebs_sample()
2014 	event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;  in intel_pt_synth_pebs_sample()
2016 	sample.cpumode = cpumode;  in intel_pt_synth_pebs_sample()
2026 			sample.time = tsc_to_perf_time(timestamp, &pt->tc);  in intel_pt_synth_pebs_sample()
2032 				     pt->synth_opts.callchain_sz, sample.ip,  in intel_pt_synth_pebs_sample()
2034 		sample.callchain = ptq->chain;  in intel_pt_synth_pebs_sample()
2043 		sample.intr_regs.abi = items->is_32_bit ?  in intel_pt_synth_pebs_sample()
2046 		sample.intr_regs.regs = regs;  in intel_pt_synth_pebs_sample()
2048 		pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);  in intel_pt_synth_pebs_sample()
2050 		intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);  in intel_pt_synth_pebs_sample()
2065 		sample.branch_stack = ptq->last_branch;  in intel_pt_synth_pebs_sample()
2069 		sample.addr = items->mem_access_address;  in intel_pt_synth_pebs_sample()
2090 				sample.weight = weight & 0xffff;  in intel_pt_synth_pebs_sample()
2091 				sample.ins_lat = items->mem_access_latency & 0xffff;  in intel_pt_synth_pebs_sample()
2093 				sample.weight = items->mem_access_latency;  in intel_pt_synth_pebs_sample()
2095 		if (!sample.weight && items->has_tsx_aux_info) {  in intel_pt_synth_pebs_sample()
2097 			sample.weight = (u32)items->tsx_aux_info;  in intel_pt_synth_pebs_sample()
2109 		sample.transaction = txn;  in intel_pt_synth_pebs_sample()
2112 	return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);  in intel_pt_synth_pebs_sample()
2644 					    struct perf_sample *sample)  in intel_pt_sample_set_pid_tid_cpu()  argument
2648 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
2649 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
2671 					    struct perf_sample *sample)  in intel_pt_process_timeless_sample()  argument
2677 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_timeless_sample()
2686 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
2687 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
2692 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)  in intel_pt_lost()  argument
2694 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,  in intel_pt_lost()
2695 				    sample->pid, sample->tid, 0, sample->time);  in intel_pt_lost()
2772 				   struct perf_sample *sample)  in intel_pt_process_switch()  argument
2776 	struct evsel *evsel = evlist__id2evsel(pt->session->evlist, sample->id);  in intel_pt_process_switch()
2781 	tid = evsel__intval(evsel, sample, "next_pid");  in intel_pt_process_switch()
2782 	cpu = sample->cpu;  in intel_pt_process_switch()
2785 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,  in intel_pt_process_switch()
2788 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_process_switch()
2796 				      struct perf_sample *sample)  in intel_pt_context_switch_in()  argument
2798 	pid_t pid = sample->pid;  in intel_pt_context_switch_in()
2799 	pid_t tid = sample->tid;  in intel_pt_context_switch_in()
2800 	int cpu = sample->cpu;  in intel_pt_context_switch_in()
2834 				   struct perf_sample *sample)  in intel_pt_context_switch()  argument
2840 	cpu = sample->cpu;  in intel_pt_context_switch()
2844 			return intel_pt_context_switch_in(pt, sample);  in intel_pt_context_switch()
2854 		pid = sample->pid;  in intel_pt_context_switch()
2855 		tid = sample->tid;  in intel_pt_context_switch()
2861 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_context_switch()
2870 					 struct perf_sample *sample)  in intel_pt_process_itrace_start()  argument
2876 		     sample->cpu, event->itrace_start.pid,  in intel_pt_process_itrace_start()
2877 		     event->itrace_start.tid, sample->time,  in intel_pt_process_itrace_start()
2878 		     perf_time_to_tsc(sample->time, &pt->tc));  in intel_pt_process_itrace_start()
2880 	return machine__set_current_tid(pt->machine, sample->cpu,  in intel_pt_process_itrace_start()
2948 				  struct perf_sample *sample,  in intel_pt_process_event()  argument
2964 	if (sample->time && sample->time != (u64)-1)  in intel_pt_process_event()
2965 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_process_event()
2977 			if (sample->aux_sample.size)  in intel_pt_process_event()
2979 								       sample);  in intel_pt_process_event()
2983 							       sample->time);  in intel_pt_process_event()
2994 		if (pt->synth_opts.add_callchain && !sample->callchain)  in intel_pt_process_event()
2995 			intel_pt_add_callchain(pt, sample);  in intel_pt_process_event()
2996 		if (pt->synth_opts.add_last_branch && !sample->branch_stack)  in intel_pt_process_event()
2997 			intel_pt_add_br_stack(pt, sample);  in intel_pt_process_event()
3003 		err = intel_pt_lost(pt, sample);  in intel_pt_process_event()
3009 		err = intel_pt_process_switch(pt, sample);  in intel_pt_process_event()
3011 		err = intel_pt_process_itrace_start(pt, event, sample);  in intel_pt_process_event()
3014 		err = intel_pt_context_switch(pt, event, sample);  in intel_pt_process_event()
3019 	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {  in intel_pt_process_event()
3021 			     event->header.type, sample->cpu, sample->time, timestamp);  in intel_pt_process_event()
3132 			       struct perf_sample *sample,  in intel_pt_queue_data()  argument
3144 	if (sample->time && sample->time != (u64)-1)  in intel_pt_queue_data()
3145 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_queue_data()
3149 	return auxtrace_queues__add_sample(&pt->queues, session, sample,  in intel_pt_queue_data()
3160 				struct perf_sample *sample __maybe_unused,  in intel_pt_event_synth()
3176 	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",  in intel_pt_synth_event()
3276 		 * We don't use the hardware index, but the sample generation  in intel_pt_synth_events()
3876 		 * actual sample ip to where the sample time is recorded.  in intel_pt_process_auxtrace_info()