243 				 struct perf_sample *sample)  in intel_pt_dump_sample()  argument
249 	intel_pt_dump(pt, sample->aux_sample.data, sample->aux_sample.size);  in intel_pt_dump_sample()
932 				   struct perf_sample *sample)  in intel_pt_add_callchain()  argument
935 							sample->pid,  in intel_pt_add_callchain()
936 							sample->tid);  in intel_pt_add_callchain()
938 	thread_stack__sample_late(thread, sample->cpu, pt->chain,  in intel_pt_add_callchain()
939 				  pt->synth_opts.callchain_sz + 1, sample->ip,  in intel_pt_add_callchain()
942 	sample->callchain = pt->chain;  in intel_pt_add_callchain()
970 				  struct perf_sample *sample)  in intel_pt_add_br_stack()  argument
973 							sample->pid,  in intel_pt_add_br_stack()
974 							sample->tid);  in intel_pt_add_br_stack()
976 	thread_stack__br_sample_late(thread, sample->cpu, pt->br_stack,  in intel_pt_add_br_stack()
977 				     pt->br_stack_sz, sample->ip,  in intel_pt_add_br_stack()
980 	sample->branch_stack = pt->br_stack;  in intel_pt_add_br_stack()
1281 				   struct perf_sample *sample)  in intel_pt_prep_a_sample()  argument
1283 	event->sample.header.type = PERF_RECORD_SAMPLE;  in intel_pt_prep_a_sample()
1284 	event->sample.header.size = sizeof(struct perf_event_header);  in intel_pt_prep_a_sample()
1286 	sample->pid = ptq->pid;  in intel_pt_prep_a_sample()
1287 	sample->tid = ptq->tid;  in intel_pt_prep_a_sample()
1288 	sample->cpu = ptq->cpu;  in intel_pt_prep_a_sample()
1289 	sample->insn_len = ptq->insn_len;  in intel_pt_prep_a_sample()
1290 	memcpy(sample->insn, ptq->insn, INTEL_PT_INSN_BUF_SZ);  in intel_pt_prep_a_sample()
1296 				   struct perf_sample *sample)  in intel_pt_prep_b_sample()  argument
1298 	intel_pt_prep_a_sample(ptq, event, sample);  in intel_pt_prep_b_sample()
1301 		sample->time = tsc_to_perf_time(ptq->timestamp, &pt->tc);  in intel_pt_prep_b_sample()
1303 	sample->ip = ptq->state->from_ip;  in intel_pt_prep_b_sample()
1304 	sample->cpumode = intel_pt_cpumode(pt, sample->ip);  in intel_pt_prep_b_sample()
1305 	sample->addr = ptq->state->to_ip;  in intel_pt_prep_b_sample()
1306 	sample->period = 1;  in intel_pt_prep_b_sample()
1307 	sample->flags = ptq->flags;  in intel_pt_prep_b_sample()
1309 	event->sample.header.misc = sample->cpumode;  in intel_pt_prep_b_sample()
1313 				 struct perf_sample *sample, u64 type)  in intel_pt_inject_event()  argument
1315 	event->header.size = perf_event__sample_event_size(sample, type, 0);  in intel_pt_inject_event()
1316 	return perf_event__synthesize_sample(event, type, 0, sample);  in intel_pt_inject_event()
1321 				      struct perf_sample *sample, u64 type)  in intel_pt_opt_inject()  argument
1326 	return intel_pt_inject_event(event, sample, type);  in intel_pt_opt_inject()
1331 					struct perf_sample *sample, u64 type)  in intel_pt_deliver_synth_event()  argument
1335 	ret = intel_pt_opt_inject(pt, event, sample, type);  in intel_pt_deliver_synth_event()
1339 	ret = perf_session__deliver_synth_event(pt->session, event, sample);  in intel_pt_deliver_synth_event()
1350 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_branch_sample()  local
1363 	intel_pt_prep_b_sample(pt, ptq, event, &sample);  in intel_pt_synth_branch_sample()
1365 	sample.id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1366 	sample.stream_id = ptq->pt->branches_id;  in intel_pt_synth_branch_sample()
1377 				.from = sample.ip,  in intel_pt_synth_branch_sample()
1378 				.to = sample.addr,  in intel_pt_synth_branch_sample()
1381 		sample.branch_stack = (struct branch_stack *)&dummy_bs;  in intel_pt_synth_branch_sample()
1384 	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_br_cyc_cnt;  in intel_pt_synth_branch_sample()
1385 	if (sample.cyc_cnt) {  in intel_pt_synth_branch_sample()
1386 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_br_insn_cnt;  in intel_pt_synth_branch_sample()
1391 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_branch_sample()
1398 				 struct perf_sample *sample)  in intel_pt_prep_sample()  argument
1400 	intel_pt_prep_b_sample(pt, ptq, event, sample);  in intel_pt_prep_sample()
1405 				     sample->ip, pt->kernel_start);  in intel_pt_prep_sample()
1406 		sample->callchain = ptq->chain;  in intel_pt_prep_sample()
1412 		sample->branch_stack = ptq->last_branch;  in intel_pt_prep_sample()
1420 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_instruction_sample()  local
1425 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_instruction_sample()
1427 	sample.id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1428 	sample.stream_id = ptq->pt->instructions_id;  in intel_pt_synth_instruction_sample()
1430 		sample.period = 1;  in intel_pt_synth_instruction_sample()
1432 		sample.period = ptq->state->tot_insn_cnt - ptq->last_insn_cnt;  in intel_pt_synth_instruction_sample()
1434 	sample.cyc_cnt = ptq->ipc_cyc_cnt - ptq->last_in_cyc_cnt;  in intel_pt_synth_instruction_sample()
1435 	if (sample.cyc_cnt) {  in intel_pt_synth_instruction_sample()
1436 		sample.insn_cnt = ptq->ipc_insn_cnt - ptq->last_in_insn_cnt;  in intel_pt_synth_instruction_sample()
1443 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_instruction_sample()
1451 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_transaction_sample()  local
1456 	intel_pt_prep_sample(pt, ptq, event, &sample);  in intel_pt_synth_transaction_sample()
1458 	sample.id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1459 	sample.stream_id = ptq->pt->transactions_id;  in intel_pt_synth_transaction_sample()
1461 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_transaction_sample()
1468 				   struct perf_sample *sample)  in intel_pt_prep_p_sample()  argument
1470 	intel_pt_prep_sample(pt, ptq, event, sample);  in intel_pt_prep_p_sample()
1476 	if (!sample->ip)  in intel_pt_prep_p_sample()
1477 		sample->flags = 0;  in intel_pt_prep_p_sample()
1484 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_ptwrite_sample()  local
1490 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_ptwrite_sample()
1492 	sample.id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1493 	sample.stream_id = ptq->pt->ptwrites_id;  in intel_pt_synth_ptwrite_sample()
1499 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_ptwrite_sample()
1500 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_ptwrite_sample()
1502 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_ptwrite_sample()
1510 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_cbr_sample()  local
1519 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_cbr_sample()
1521 	sample.id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1522 	sample.stream_id = ptq->pt->cbr_id;  in intel_pt_synth_cbr_sample()
1529 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_cbr_sample()
1530 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_cbr_sample()
1532 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_cbr_sample()
1540 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_mwait_sample()  local
1546 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_mwait_sample()
1548 	sample.id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1549 	sample.stream_id = ptq->pt->mwait_id;  in intel_pt_synth_mwait_sample()
1554 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_mwait_sample()
1555 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_mwait_sample()
1557 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_mwait_sample()
1565 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwre_sample()  local
1571 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwre_sample()
1573 	sample.id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1574 	sample.stream_id = ptq->pt->pwre_id;  in intel_pt_synth_pwre_sample()
1579 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwre_sample()
1580 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwre_sample()
1582 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwre_sample()
1590 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_exstop_sample()  local
1596 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_exstop_sample()
1598 	sample.id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1599 	sample.stream_id = ptq->pt->exstop_id;  in intel_pt_synth_exstop_sample()
1604 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_exstop_sample()
1605 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_exstop_sample()
1607 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_exstop_sample()
1615 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pwrx_sample()  local
1621 	intel_pt_prep_p_sample(pt, ptq, event, &sample);  in intel_pt_synth_pwrx_sample()
1623 	sample.id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1624 	sample.stream_id = ptq->pt->pwrx_id;  in intel_pt_synth_pwrx_sample()
1629 	sample.raw_size = perf_synth__raw_size(raw);  in intel_pt_synth_pwrx_sample()
1630 	sample.raw_data = perf_synth__raw_data(&raw);  in intel_pt_synth_pwrx_sample()
1632 	return intel_pt_deliver_synth_event(pt, event, &sample,  in intel_pt_synth_pwrx_sample()
1766 	struct perf_sample sample = { .ip = 0, };  in intel_pt_synth_pebs_sample()  local
1773 	u64 regs[8 * sizeof(sample.intr_regs.mask)];  in intel_pt_synth_pebs_sample()
1778 	intel_pt_prep_a_sample(ptq, event, &sample);  in intel_pt_synth_pebs_sample()
1780 	sample.id = id;  in intel_pt_synth_pebs_sample()
1781 	sample.stream_id = id;  in intel_pt_synth_pebs_sample()
1784 		sample.period = evsel->core.attr.sample_period;  in intel_pt_synth_pebs_sample()
1788 		sample.ip = items->ip;  in intel_pt_synth_pebs_sample()
1790 		sample.ip = items->rip;  in intel_pt_synth_pebs_sample()
1792 		sample.ip = ptq->state->from_ip;  in intel_pt_synth_pebs_sample()
1795 	cpumode = sample.ip < ptq->pt->kernel_start ?  in intel_pt_synth_pebs_sample()
1799 	event->sample.header.misc = cpumode | PERF_RECORD_MISC_EXACT_IP;  in intel_pt_synth_pebs_sample()
1801 	sample.cpumode = cpumode;  in intel_pt_synth_pebs_sample()
1811 			sample.time = tsc_to_perf_time(timestamp, &pt->tc);  in intel_pt_synth_pebs_sample()
1817 				     pt->synth_opts.callchain_sz, sample.ip,  in intel_pt_synth_pebs_sample()
1819 		sample.callchain = ptq->chain;  in intel_pt_synth_pebs_sample()
1828 		sample.intr_regs.abi = items->is_32_bit ?  in intel_pt_synth_pebs_sample()
1831 		sample.intr_regs.regs = regs;  in intel_pt_synth_pebs_sample()
1833 		pos = intel_pt_add_gp_regs(&sample.intr_regs, regs, items, regs_mask);  in intel_pt_synth_pebs_sample()
1835 		intel_pt_add_xmm(&sample.intr_regs, pos, items, regs_mask);  in intel_pt_synth_pebs_sample()
1850 		sample.branch_stack = ptq->last_branch;  in intel_pt_synth_pebs_sample()
1854 		sample.addr = items->mem_access_address;  in intel_pt_synth_pebs_sample()
1862 			sample.weight = items->mem_access_latency;  in intel_pt_synth_pebs_sample()
1863 		if (!sample.weight && items->has_tsx_aux_info) {  in intel_pt_synth_pebs_sample()
1865 			sample.weight = (u32)items->tsx_aux_info;  in intel_pt_synth_pebs_sample()
1877 		sample.transaction = txn;  in intel_pt_synth_pebs_sample()
1880 	return intel_pt_deliver_synth_event(pt, event, &sample, sample_type);  in intel_pt_synth_pebs_sample()
2393 					    struct perf_sample *sample)  in intel_pt_sample_set_pid_tid_cpu()  argument
2397 	ptq->pid = sample->pid;  in intel_pt_sample_set_pid_tid_cpu()
2398 	ptq->tid = sample->tid;  in intel_pt_sample_set_pid_tid_cpu()
2420 					    struct perf_sample *sample)  in intel_pt_process_timeless_sample()  argument
2426 	queue = auxtrace_queues__sample_queue(&pt->queues, sample, pt->session);  in intel_pt_process_timeless_sample()
2435 	ptq->time = sample->time;  in intel_pt_process_timeless_sample()
2436 	intel_pt_sample_set_pid_tid_cpu(ptq, queue, sample);  in intel_pt_process_timeless_sample()
2441 static int intel_pt_lost(struct intel_pt *pt, struct perf_sample *sample)  in intel_pt_lost()  argument
2443 	return intel_pt_synth_error(pt, INTEL_PT_ERR_LOST, sample->cpu,  in intel_pt_lost()
2444 				    sample->pid, sample->tid, 0, sample->time);  in intel_pt_lost()
2521 				   struct perf_sample *sample)  in intel_pt_process_switch()  argument
2527 	evsel = perf_evlist__id2evsel(pt->session->evlist, sample->id);  in intel_pt_process_switch()
2531 	tid = evsel__intval(evsel, sample, "next_pid");  in intel_pt_process_switch()
2532 	cpu = sample->cpu;  in intel_pt_process_switch()
2535 		     cpu, tid, sample->time, perf_time_to_tsc(sample->time,  in intel_pt_process_switch()
2538 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_process_switch()
2546 				      struct perf_sample *sample)  in intel_pt_context_switch_in()  argument
2548 	pid_t pid = sample->pid;  in intel_pt_context_switch_in()
2549 	pid_t tid = sample->tid;  in intel_pt_context_switch_in()
2550 	int cpu = sample->cpu;  in intel_pt_context_switch_in()
2584 				   struct perf_sample *sample)  in intel_pt_context_switch()  argument
2590 	cpu = sample->cpu;  in intel_pt_context_switch()
2594 			return intel_pt_context_switch_in(pt, sample);  in intel_pt_context_switch()
2604 		pid = sample->pid;  in intel_pt_context_switch()
2605 		tid = sample->tid;  in intel_pt_context_switch()
2611 	ret = intel_pt_sync_switch(pt, cpu, tid, sample->time);  in intel_pt_context_switch()
2620 					 struct perf_sample *sample)  in intel_pt_process_itrace_start()  argument
2626 		     sample->cpu, event->itrace_start.pid,  in intel_pt_process_itrace_start()
2627 		     event->itrace_start.tid, sample->time,  in intel_pt_process_itrace_start()
2628 		     perf_time_to_tsc(sample->time, &pt->tc));  in intel_pt_process_itrace_start()
2630 	return machine__set_current_tid(pt->machine, sample->cpu,  in intel_pt_process_itrace_start()
2698 				  struct perf_sample *sample,  in intel_pt_process_event()  argument
2714 	if (sample->time && sample->time != (u64)-1)  in intel_pt_process_event()
2715 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_process_event()
2727 			if (sample->aux_sample.size)  in intel_pt_process_event()
2729 								       sample);  in intel_pt_process_event()
2733 							       sample->time);  in intel_pt_process_event()
2742 		if (pt->synth_opts.add_callchain && !sample->callchain)  in intel_pt_process_event()
2743 			intel_pt_add_callchain(pt, sample);  in intel_pt_process_event()
2744 		if (pt->synth_opts.add_last_branch && !sample->branch_stack)  in intel_pt_process_event()
2745 			intel_pt_add_br_stack(pt, sample);  in intel_pt_process_event()
2751 		err = intel_pt_lost(pt, sample);  in intel_pt_process_event()
2757 		err = intel_pt_process_switch(pt, sample);  in intel_pt_process_event()
2759 		err = intel_pt_process_itrace_start(pt, event, sample);  in intel_pt_process_event()
2762 		err = intel_pt_context_switch(pt, event, sample);  in intel_pt_process_event()
2767 	if (intel_pt_enable_logging && intel_pt_log_events(pt, sample->time)) {  in intel_pt_process_event()
2769 			     event->header.type, sample->cpu, sample->time, timestamp);  in intel_pt_process_event()
2879 			       struct perf_sample *sample,  in intel_pt_queue_data()  argument
2891 	if (sample->time && sample->time != (u64)-1)  in intel_pt_queue_data()
2892 		timestamp = perf_time_to_tsc(sample->time, &pt->tc);  in intel_pt_queue_data()
2896 	return auxtrace_queues__add_sample(&pt->queues, session, sample,  in intel_pt_queue_data()
2907 				struct perf_sample *sample __maybe_unused,  in intel_pt_event_synth()
2923 	pr_debug("Synthesizing '%s' event with id %" PRIu64 " sample type %#" PRIx64 "\n",  in intel_pt_synth_event()
3023 		 * We don't use the hardware index, but the sample generation  in intel_pt_synth_events()
3530 		 * actual sample ip to where the sample time is recorded.  in intel_pt_process_auxtrace_info()