21 #include "cs-etm.h"
22 #include "cs-etm-decoder/cs-etm-decoder.h"
84 	struct cs_etm_auxtrace *etm;  member
100 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm);
101 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm);
102 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
113  * encode the etm queue number as the upper 16 bit and the channel as
239 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__init_traceid_queue()  local
243 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];  in cs_etm__init_traceid_queue()
256 	if (etm->synth_opts.last_branch) {  in cs_etm__init_traceid_queue()
259 		sz += etm->synth_opts.last_branch_sz *  in cs_etm__init_traceid_queue()
291 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__etmq_get_traceid_queue()  local
293 	if (etm->timeless_decoding)  in cs_etm__etmq_get_traceid_queue()
369 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,  in cs_etm__packet_swap()  argument
374 	if (etm->sample_branches || etm->synth_opts.last_branch ||  in cs_etm__packet_swap()
375 	    etm->sample_instructions) {  in cs_etm__packet_swap()
400 					  struct cs_etm_auxtrace *etm, int idx,  in cs_etm__set_trace_param_etmv3()  argument
403 	u64 **metadata = etm->metadata;  in cs_etm__set_trace_param_etmv3()
411 					  struct cs_etm_auxtrace *etm, int idx)  in cs_etm__set_trace_param_etmv4()  argument
413 	u64 **metadata = etm->metadata;  in cs_etm__set_trace_param_etmv4()
425 				     struct cs_etm_auxtrace *etm)  in cs_etm__init_trace_params()  argument
431 	for (i = 0; i < etm->num_cpu; i++) {  in cs_etm__init_trace_params()
432 		architecture = etm->metadata[i][CS_ETM_MAGIC];  in cs_etm__init_trace_params()
436 			etmidr = etm->metadata[i][CS_ETM_ETMIDR];  in cs_etm__init_trace_params()
437 			cs_etm__set_trace_param_etmv3(t_params, etm, i, etmidr);  in cs_etm__init_trace_params()
440 			cs_etm__set_trace_param_etmv4(t_params, etm, i);  in cs_etm__init_trace_params()
472 static void cs_etm__dump_event(struct cs_etm_auxtrace *etm,  in cs_etm__dump_event()  argument
484 		     ". ... CoreSight ETM Trace data: size %zu bytes\n",  in cs_etm__dump_event()
488 	t_params = zalloc(sizeof(*t_params) * etm->num_cpu);  in cs_etm__dump_event()
493 	if (cs_etm__init_trace_params(t_params, etm))  in cs_etm__dump_event()
501 	decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);  in cs_etm__dump_event()
528 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__flush_events()  local
537 	ret = cs_etm__update_queues(etm);  in cs_etm__flush_events()
542 	if (etm->timeless_decoding)  in cs_etm__flush_events()
543 		return cs_etm__process_timeless_queues(etm, -1);  in cs_etm__flush_events()
545 	return cs_etm__process_queues(etm);  in cs_etm__flush_events()
651 	machine = etmq->etm->machine;  in cs_etm__cpu_mode()
653 	if (address >= etmq->etm->kernel_start) {  in cs_etm__cpu_mode()
682 	machine = etmq->etm->machine;  in cs_etm__mem_access()
692 		thread = etmq->etm->unknown_thread;  in cs_etm__mem_access()
714 static struct cs_etm_queue *cs_etm__alloc_queue(struct cs_etm_auxtrace *etm)  in cs_etm__alloc_queue()  argument
729 	t_params = zalloc(sizeof(*t_params) * etm->num_cpu);  in cs_etm__alloc_queue()
734 	if (cs_etm__init_trace_params(t_params, etm))  in cs_etm__alloc_queue()
742 	etmq->decoder = cs_etm_decoder__new(etm->num_cpu, &d_params, t_params);  in cs_etm__alloc_queue()
768 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,  in cs_etm__setup_queue()  argument
781 	etmq = cs_etm__alloc_queue(etm);  in cs_etm__setup_queue()
789 	etmq->etm = etm;  in cs_etm__setup_queue()
793 	if (etm->timeless_decoding)  in cs_etm__setup_queue()
853 	ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);  in cs_etm__setup_queue()
858 static int cs_etm__setup_queues(struct cs_etm_auxtrace *etm)  in cs_etm__setup_queues()  argument
863 	if (!etm->kernel_start)  in cs_etm__setup_queues()
864 		etm->kernel_start = machine__kernel_start(etm->machine);  in cs_etm__setup_queues()
866 	for (i = 0; i < etm->queues.nr_queues; i++) {  in cs_etm__setup_queues()
867 		ret = cs_etm__setup_queue(etm, &etm->queues.queue_array[i], i);  in cs_etm__setup_queues()
875 static int cs_etm__update_queues(struct cs_etm_auxtrace *etm)  in cs_etm__update_queues()  argument
877 	if (etm->queues.new_data) {  in cs_etm__update_queues()
878 		etm->queues.new_data = false;  in cs_etm__update_queues()
879 		return cs_etm__setup_queues(etm);  in cs_etm__update_queues()
910 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
922 	if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {  in cs_etm__copy_last_branch_rb()
1003 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1018 	if (bs->nr < etmq->etm->synth_opts.last_branch_sz)  in cs_etm__update_last_branch_rb()
1037 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];  in cs_etm__get_trace()
1054 		int fd = perf_data__fd(etmq->etm->session->data);  in cs_etm__get_trace()
1072 static void cs_etm__set_pid_tid_cpu(struct cs_etm_auxtrace *etm,  in cs_etm__set_pid_tid_cpu()  argument
1076 		tidq->thread = machine__find_thread(etm->machine, -1,  in cs_etm__set_pid_tid_cpu()
1087 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__etmq_set_tid()  local
1097 	err = machine__set_current_tid(etm->machine, cpu, tid, tid);  in cs_etm__etmq_set_tid()
1104 	cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__etmq_set_tid()
1110 	return !!etmq->etm->timeless_decoding;  in cs_etm__etmq_is_timeless()
1147 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__synth_instruction_sample()  local
1158 	sample.id = etmq->etm->instructions_id;  in cs_etm__synth_instruction_sample()
1159 	sample.stream_id = etmq->etm->instructions_id;  in cs_etm__synth_instruction_sample()
1167 	if (etm->synth_opts.last_branch)  in cs_etm__synth_instruction_sample()
1170 	if (etm->synth_opts.inject) {  in cs_etm__synth_instruction_sample()
1172 					   etm->instructions_sample_type);  in cs_etm__synth_instruction_sample()
1177 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);  in cs_etm__synth_instruction_sample()
1181 			"CS ETM Trace: failed to deliver instruction event, error %d\n",  in cs_etm__synth_instruction_sample()
1188  * The cs etm packet encodes an instruction range between a branch target
1195 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__synth_branch_sample()  local
1215 	sample.id = etmq->etm->branches_id;  in cs_etm__synth_branch_sample()
1216 	sample.stream_id = etmq->etm->branches_id;  in cs_etm__synth_branch_sample()
1228 	if (etm->synth_opts.last_branch) {  in cs_etm__synth_branch_sample()
1240 	if (etm->synth_opts.inject) {  in cs_etm__synth_branch_sample()
1242 					   etm->branches_sample_type);  in cs_etm__synth_branch_sample()
1247 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);  in cs_etm__synth_branch_sample()
1251 		"CS ETM Trace: failed to deliver instruction event, error %d\n",  in cs_etm__synth_branch_sample()
1286 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,  in cs_etm__synth_events()  argument
1297 		if (evsel->core.attr.type == etm->pmu_type) {  in cs_etm__synth_events()
1314 	if (etm->timeless_decoding)  in cs_etm__synth_events()
1333 	if (etm->synth_opts.branches) {  in cs_etm__synth_events()
1340 		etm->sample_branches = true;  in cs_etm__synth_events()
1341 		etm->branches_sample_type = attr.sample_type;  in cs_etm__synth_events()
1342 		etm->branches_id = id;  in cs_etm__synth_events()
1347 	if (etm->synth_opts.last_branch) {  in cs_etm__synth_events()
1357 	if (etm->synth_opts.instructions) {  in cs_etm__synth_events()
1359 		attr.sample_period = etm->synth_opts.period;  in cs_etm__synth_events()
1360 		etm->instructions_sample_period = attr.sample_period;  in cs_etm__synth_events()
1364 		etm->sample_instructions = true;  in cs_etm__synth_events()
1365 		etm->instructions_sample_type = attr.sample_type;  in cs_etm__synth_events()
1366 		etm->instructions_id = id;  in cs_etm__synth_events()
1376 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__sample()  local
1390 	if (etm->synth_opts.last_branch &&  in cs_etm__sample()
1395 	if (etm->sample_instructions &&  in cs_etm__sample()
1396 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1422 		 * every etm->instructions_sample_period instructions - as  in cs_etm__sample()
1424 		 * last sample before the current etm packet, n+1 to n+3  in cs_etm__sample()
1425 		 * samples are generated from the current etm packet.  in cs_etm__sample()
1428 		 * instructions in the current etm packet.  in cs_etm__sample()
1432 		 * previous etm packet.  This will always be less than  in cs_etm__sample()
1433 		 * etm->instructions_sample_period.  in cs_etm__sample()
1447 		 * etm->instructions_sample_period.  in cs_etm__sample()
1449 		u64 offset = etm->instructions_sample_period - instrs_prev;  in cs_etm__sample()
1453 		if (etm->synth_opts.last_branch)  in cs_etm__sample()
1457 				etm->instructions_sample_period) {  in cs_etm__sample()
1468 				etm->instructions_sample_period);  in cs_etm__sample()
1472 			offset += etm->instructions_sample_period;  in cs_etm__sample()
1474 				etm->instructions_sample_period;  in cs_etm__sample()
1478 	if (etm->sample_branches) {  in cs_etm__sample()
1497 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1525 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__flush()  local
1531 	if (etmq->etm->synth_opts.last_branch &&  in cs_etm__flush()
1557 	if (etm->sample_branches &&  in cs_etm__flush()
1565 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1568 	if (etm->synth_opts.last_branch)  in cs_etm__flush()
1588 	if (etmq->etm->synth_opts.last_branch &&  in cs_etm__end_block()
2078 			pr_err("CS ETM Trace: empty packet\n");  in cs_etm__process_traceid_queue()
2148 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,  in cs_etm__process_timeless_queues()  argument
2152 	struct auxtrace_queues *queues = &etm->queues;  in cs_etm__process_timeless_queues()
2155 		struct auxtrace_queue *queue = &etm->queues.queue_array[i];  in cs_etm__process_timeless_queues()
2169 			cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__process_timeless_queues()
2177 static int cs_etm__process_queues(struct cs_etm_auxtrace *etm)  in cs_etm__process_queues()  argument
2188 		if (!etm->heap.heap_cnt)  in cs_etm__process_queues()
2192 		cs_queue_nr = etm->heap.heap_array[0].queue_nr;  in cs_etm__process_queues()
2195 		queue = &etm->queues.queue_array[queue_nr];  in cs_etm__process_queues()
2202 		auxtrace_heap__pop(&etm->heap);  in cs_etm__process_queues()
2269 		ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp);  in cs_etm__process_queues()
2276 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,  in cs_etm__process_itrace_start()  argument
2281 	if (etm->timeless_decoding)  in cs_etm__process_itrace_start()
2288 	th = machine__findnew_thread(etm->machine,  in cs_etm__process_itrace_start()
2299 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,  in cs_etm__process_switch_cpu_wide()  argument
2309 	if (etm->timeless_decoding)  in cs_etm__process_switch_cpu_wide()
2324 	th = machine__findnew_thread(etm->machine,  in cs_etm__process_switch_cpu_wide()
2342 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__process_event()  local
2350 		pr_err("CoreSight ETM Trace requires ordered events\n");  in cs_etm__process_event()
2359 	if (timestamp || etm->timeless_decoding) {  in cs_etm__process_event()
2360 		err = cs_etm__update_queues(etm);  in cs_etm__process_event()
2365 	if (etm->timeless_decoding &&  in cs_etm__process_event()
2367 		return cs_etm__process_timeless_queues(etm,  in cs_etm__process_event()
2371 		return cs_etm__process_itrace_start(etm, event);  in cs_etm__process_event()
2373 		return cs_etm__process_switch_cpu_wide(etm, event);  in cs_etm__process_event()
2375 	if (!etm->timeless_decoding &&  in cs_etm__process_event()
2377 		return cs_etm__process_queues(etm);  in cs_etm__process_event()
2386 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__process_auxtrace_event()  local
2389 	if (!etm->data_queued) {  in cs_etm__process_auxtrace_event()
2404 		err = auxtrace_queues__add_event(&etm->queues, session,  in cs_etm__process_auxtrace_event()
2411 				cs_etm__dump_event(etm, buffer);  in cs_etm__process_auxtrace_event()
2419 static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm)  in cs_etm__is_timeless_decoding()  argument
2422 	struct evlist *evlist = etm->session->evlist;  in cs_etm__is_timeless_decoding()
2488 	struct cs_etm_auxtrace *etm = NULL;  in cs_etm__process_auxtrace_info()  local
2523 	/* Extract header information - see cs-etm.h for format */  in cs_etm__process_auxtrace_info()
2615 	etm = zalloc(sizeof(*etm));  in cs_etm__process_auxtrace_info()
2617 	if (!etm) {  in cs_etm__process_auxtrace_info()
2622 	err = auxtrace_queues__init(&etm->queues);  in cs_etm__process_auxtrace_info()
2626 	etm->session = session;  in cs_etm__process_auxtrace_info()
2627 	etm->machine = &session->machines.host;  in cs_etm__process_auxtrace_info()
2629 	etm->num_cpu = num_cpu;  in cs_etm__process_auxtrace_info()
2630 	etm->pmu_type = pmu_type;  in cs_etm__process_auxtrace_info()
2631 	etm->snapshot_mode = (hdr[CS_ETM_SNAPSHOT] != 0);  in cs_etm__process_auxtrace_info()
2632 	etm->metadata = metadata;  in cs_etm__process_auxtrace_info()
2633 	etm->auxtrace_type = auxtrace_info->type;  in cs_etm__process_auxtrace_info()
2634 	etm->timeless_decoding = cs_etm__is_timeless_decoding(etm);  in cs_etm__process_auxtrace_info()
2636 	etm->auxtrace.process_event = cs_etm__process_event;  in cs_etm__process_auxtrace_info()
2637 	etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;  in cs_etm__process_auxtrace_info()
2638 	etm->auxtrace.flush_events = cs_etm__flush_events;  in cs_etm__process_auxtrace_info()
2639 	etm->auxtrace.free_events = cs_etm__free_events;  in cs_etm__process_auxtrace_info()
2640 	etm->auxtrace.free = cs_etm__free;  in cs_etm__process_auxtrace_info()
2641 	etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;  in cs_etm__process_auxtrace_info()
2642 	session->auxtrace = &etm->auxtrace;  in cs_etm__process_auxtrace_info()
2644 	etm->unknown_thread = thread__new(999999999, 999999999);  in cs_etm__process_auxtrace_info()
2645 	if (!etm->unknown_thread) {  in cs_etm__process_auxtrace_info()
2654 	INIT_LIST_HEAD(&etm->unknown_thread->node);  in cs_etm__process_auxtrace_info()
2656 	err = thread__set_comm(etm->unknown_thread, "unknown", 0);  in cs_etm__process_auxtrace_info()
2660 	if (thread__init_maps(etm->unknown_thread, etm->machine)) {  in cs_etm__process_auxtrace_info()
2671 		etm->synth_opts = *session->itrace_synth_opts;  in cs_etm__process_auxtrace_info()
2673 		itrace_synth_opts__set_default(&etm->synth_opts,  in cs_etm__process_auxtrace_info()
2675 		etm->synth_opts.callchain = false;  in cs_etm__process_auxtrace_info()
2678 	err = cs_etm__synth_events(etm, session);  in cs_etm__process_auxtrace_info()
2682 	err = auxtrace_queues__process_index(&etm->queues, session);  in cs_etm__process_auxtrace_info()
2686 	etm->data_queued = etm->queues.populated;  in cs_etm__process_auxtrace_info()
2691 	thread__zput(etm->unknown_thread);  in cs_etm__process_auxtrace_info()
2693 	auxtrace_queues__free(&etm->queues);  in cs_etm__process_auxtrace_info()
2696 	zfree(&etm);  in cs_etm__process_auxtrace_info()