262 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_packet_queues()  local
267 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_packet_queues()
268 		cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__clear_all_packet_queues()
273 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__init_traceid_queue()  argument
280 	cs_etm__clear_packet_queue(&tidq->packet_queue);  in cs_etm__init_traceid_queue()
283 	tidq->tid = queue->tid;  in cs_etm__init_traceid_queue()
284 	tidq->pid = -1;  in cs_etm__init_traceid_queue()
285 	tidq->trace_chan_id = trace_chan_id;  in cs_etm__init_traceid_queue()
287 	tidq->packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
288 	if (!tidq->packet)  in cs_etm__init_traceid_queue()
291 	tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet));  in cs_etm__init_traceid_queue()
292 	if (!tidq->prev_packet)  in cs_etm__init_traceid_queue()
300 		tidq->last_branch = zalloc(sz);  in cs_etm__init_traceid_queue()
301 		if (!tidq->last_branch)  in cs_etm__init_traceid_queue()
303 		tidq->last_branch_rb = zalloc(sz);  in cs_etm__init_traceid_queue()
304 		if (!tidq->last_branch_rb)  in cs_etm__init_traceid_queue()
308 	tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE);  in cs_etm__init_traceid_queue()
309 	if (!tidq->event_buf)  in cs_etm__init_traceid_queue()
315 	zfree(&tidq->last_branch_rb);  in cs_etm__init_traceid_queue()
316 	zfree(&tidq->last_branch);  in cs_etm__init_traceid_queue()
317 	zfree(&tidq->prev_packet);  in cs_etm__init_traceid_queue()
318 	zfree(&tidq->packet);  in cs_etm__init_traceid_queue()
329 	struct cs_etm_traceid_queue *tidq, **traceid_queues;  in cs_etm__etmq_get_traceid_queue()  local
348 	tidq = malloc(sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
349 	if (!tidq)  in cs_etm__etmq_get_traceid_queue()
352 	memset(tidq, 0, sizeof(*tidq));  in cs_etm__etmq_get_traceid_queue()
364 	if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id))  in cs_etm__etmq_get_traceid_queue()
380 	traceid_queues[idx] = tidq;  in cs_etm__etmq_get_traceid_queue()
391 	free(tidq);  in cs_etm__etmq_get_traceid_queue()
399 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_get_packet_queue()  local
401 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_get_packet_queue()
402 	if (tidq)  in cs_etm__etmq_get_packet_queue()
403 		return &tidq->packet_queue;  in cs_etm__etmq_get_packet_queue()
409 				struct cs_etm_traceid_queue *tidq)  in cs_etm__packet_swap()  argument
419 		tmp = tidq->packet;  in cs_etm__packet_swap()
420 		tidq->packet = tidq->prev_packet;  in cs_etm__packet_swap()
421 		tidq->prev_packet = tmp;  in cs_etm__packet_swap()
582 	struct cs_etm_traceid_queue *tidq;  in cs_etm__free_traceid_queues()  local
590 		tidq = etmq->traceid_queues[idx];  in cs_etm__free_traceid_queues()
591 		thread__zput(tidq->thread);  in cs_etm__free_traceid_queues()
592 		zfree(&tidq->event_buf);  in cs_etm__free_traceid_queues()
593 		zfree(&tidq->last_branch);  in cs_etm__free_traceid_queues()
594 		zfree(&tidq->last_branch_rb);  in cs_etm__free_traceid_queues()
595 		zfree(&tidq->prev_packet);  in cs_etm__free_traceid_queues()
596 		zfree(&tidq->packet);  in cs_etm__free_traceid_queues()
597 		zfree(&tidq);  in cs_etm__free_traceid_queues()
706 	struct cs_etm_traceid_queue *tidq;  in cs_etm__mem_access()  local
713 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__mem_access()
714 	if (!tidq)  in cs_etm__mem_access()
717 	thread = tidq->thread;  in cs_etm__mem_access()
911 				 struct cs_etm_traceid_queue *tidq)  in cs_etm__copy_last_branch_rb()  argument
913 	struct branch_stack *bs_src = tidq->last_branch_rb;  in cs_etm__copy_last_branch_rb()
914 	struct branch_stack *bs_dst = tidq->last_branch;  in cs_etm__copy_last_branch_rb()
934 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
936 	       &bs_src->entries[tidq->last_branch_pos],  in cs_etm__copy_last_branch_rb()
949 		       sizeof(struct branch_entry) * tidq->last_branch_pos);  in cs_etm__copy_last_branch_rb()
954 void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq)  in cs_etm__reset_last_branch_rb()  argument
956 	tidq->last_branch_pos = 0;  in cs_etm__reset_last_branch_rb()
957 	tidq->last_branch_rb->nr = 0;  in cs_etm__reset_last_branch_rb()
1015 					  struct cs_etm_traceid_queue *tidq)  in cs_etm__update_last_branch_rb()  argument
1017 	struct branch_stack *bs = tidq->last_branch_rb;  in cs_etm__update_last_branch_rb()
1026 	if (!tidq->last_branch_pos)  in cs_etm__update_last_branch_rb()
1027 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1029 	tidq->last_branch_pos -= 1;  in cs_etm__update_last_branch_rb()
1031 	be       = &bs->entries[tidq->last_branch_pos];  in cs_etm__update_last_branch_rb()
1032 	be->from = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__update_last_branch_rb()
1033 	be->to	 = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__update_last_branch_rb()
1097 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_pid_tid_cpu()  argument
1099 	if ((!tidq->thread) && (tidq->tid != -1))  in cs_etm__set_pid_tid_cpu()
1100 		tidq->thread = machine__find_thread(etm->machine, -1,  in cs_etm__set_pid_tid_cpu()
1101 						    tidq->tid);  in cs_etm__set_pid_tid_cpu()
1103 	if (tidq->thread)  in cs_etm__set_pid_tid_cpu()
1104 		tidq->pid = tidq->thread->pid_;  in cs_etm__set_pid_tid_cpu()
1112 	struct cs_etm_traceid_queue *tidq;  in cs_etm__etmq_set_tid()  local
1114 	tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__etmq_set_tid()
1115 	if (!tidq)  in cs_etm__etmq_set_tid()
1125 	tidq->tid = tid;  in cs_etm__etmq_set_tid()
1126 	thread__zput(tidq->thread);  in cs_etm__etmq_set_tid()
1128 	cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__etmq_set_tid()
1167 					    struct cs_etm_traceid_queue *tidq,  in cs_etm__synth_instruction_sample()  argument
1172 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_instruction_sample()
1182 	sample.pid = tidq->pid;  in cs_etm__synth_instruction_sample()
1183 	sample.tid = tidq->tid;  in cs_etm__synth_instruction_sample()
1187 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_instruction_sample()
1188 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_instruction_sample()
1191 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->packet, &sample);  in cs_etm__synth_instruction_sample()
1194 		sample.branch_stack = tidq->last_branch;  in cs_etm__synth_instruction_sample()
1218 				       struct cs_etm_traceid_queue *tidq)  in cs_etm__synth_branch_sample()  argument
1223 	union perf_event *event = tidq->event_buf;  in cs_etm__synth_branch_sample()
1231 	ip = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__synth_branch_sample()
1240 	sample.pid = tidq->pid;  in cs_etm__synth_branch_sample()
1241 	sample.tid = tidq->tid;  in cs_etm__synth_branch_sample()
1242 	sample.addr = cs_etm__first_executed_instr(tidq->packet);  in cs_etm__synth_branch_sample()
1246 	sample.cpu = tidq->packet->cpu;  in cs_etm__synth_branch_sample()
1247 	sample.flags = tidq->prev_packet->flags;  in cs_etm__synth_branch_sample()
1250 	cs_etm__copy_insn(etmq, tidq->trace_chan_id, tidq->prev_packet,  in cs_etm__synth_branch_sample()
1402 			  struct cs_etm_traceid_queue *tidq)  in cs_etm__sample()  argument
1406 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__sample()
1410 	instrs_prev = tidq->period_instructions;  in cs_etm__sample()
1412 	tidq->period_instructions += tidq->packet->instr_count;  in cs_etm__sample()
1419 	    tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1420 	    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1421 		cs_etm__update_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1424 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1482 			cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__sample()
1484 		while (tidq->period_instructions >=  in cs_etm__sample()
1493 						  tidq->packet, offset - 1);  in cs_etm__sample()
1495 				etmq, tidq, addr,  in cs_etm__sample()
1501 			tidq->period_instructions -=  in cs_etm__sample()
1510 		if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY)  in cs_etm__sample()
1514 		if (tidq->prev_packet->sample_type == CS_ETM_RANGE &&  in cs_etm__sample()
1515 		    tidq->prev_packet->last_instr_taken_branch)  in cs_etm__sample()
1519 			ret = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__sample()
1525 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1530 static int cs_etm__exception(struct cs_etm_traceid_queue *tidq)  in cs_etm__exception()  argument
1543 	if (tidq->prev_packet->sample_type == CS_ETM_RANGE)  in cs_etm__exception()
1544 		tidq->prev_packet->last_instr_taken_branch = true;  in cs_etm__exception()
1550 			 struct cs_etm_traceid_queue *tidq)  in cs_etm__flush()  argument
1556 	if (tidq->prev_packet->sample_type == CS_ETM_EMPTY)  in cs_etm__flush()
1560 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1564 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__flush()
1573 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__flush()
1576 			etmq, tidq, addr,  in cs_etm__flush()
1577 			tidq->period_instructions);  in cs_etm__flush()
1581 		tidq->period_instructions = 0;  in cs_etm__flush()
1586 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__flush()
1587 		err = cs_etm__synth_branch_sample(etmq, tidq);  in cs_etm__flush()
1593 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1597 		cs_etm__reset_last_branch_rb(tidq);  in cs_etm__flush()
1603 			     struct cs_etm_traceid_queue *tidq)  in cs_etm__end_block()  argument
1617 	    tidq->prev_packet->sample_type == CS_ETM_RANGE) {  in cs_etm__end_block()
1621 		cs_etm__copy_last_branch_rb(etmq, tidq);  in cs_etm__end_block()
1627 		addr = cs_etm__last_executed_instr(tidq->prev_packet);  in cs_etm__end_block()
1630 			etmq, tidq, addr,  in cs_etm__end_block()
1631 			tidq->period_instructions);  in cs_etm__end_block()
1635 		tidq->period_instructions = 0;  in cs_etm__end_block()
1739 			       struct cs_etm_traceid_queue *tidq, u64 magic)  in cs_etm__is_syscall()  argument
1741 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_syscall()
1742 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_syscall()
1743 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_syscall()
1764 static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq,  in cs_etm__is_async_exception()  argument
1767 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_async_exception()
1791 				      struct cs_etm_traceid_queue *tidq,  in cs_etm__is_sync_exception()  argument
1794 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__is_sync_exception()
1795 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__is_sync_exception()
1796 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__is_sync_exception()
1840 				    struct cs_etm_traceid_queue *tidq)  in cs_etm__set_sample_flags()  argument
1842 	struct cs_etm_packet *packet = tidq->packet;  in cs_etm__set_sample_flags()
1843 	struct cs_etm_packet *prev_packet = tidq->prev_packet;  in cs_etm__set_sample_flags()
1844 	u8 trace_chan_id = tidq->trace_chan_id;  in cs_etm__set_sample_flags()
1947 		if (cs_etm__is_syscall(etmq, tidq, magic))  in cs_etm__set_sample_flags()
1955 		else if (cs_etm__is_async_exception(tidq, magic))  in cs_etm__set_sample_flags()
1964 		else if (cs_etm__is_sync_exception(etmq, tidq, magic))  in cs_etm__set_sample_flags()
2047 					 struct cs_etm_traceid_queue *tidq)  in cs_etm__process_traceid_queue()  argument
2052 	packet_queue = &tidq->packet_queue;  in cs_etm__process_traceid_queue()
2057 						 tidq->packet);  in cs_etm__process_traceid_queue()
2072 		ret = cs_etm__set_sample_flags(etmq, tidq);  in cs_etm__process_traceid_queue()
2076 		switch (tidq->packet->sample_type) {  in cs_etm__process_traceid_queue()
2083 			cs_etm__sample(etmq, tidq);  in cs_etm__process_traceid_queue()
2092 			cs_etm__exception(tidq);  in cs_etm__process_traceid_queue()
2099 			cs_etm__flush(etmq, tidq);  in cs_etm__process_traceid_queue()
2120 	struct cs_etm_traceid_queue *tidq;  in cs_etm__clear_all_traceid_queues()  local
2125 		tidq = etmq->traceid_queues[idx];  in cs_etm__clear_all_traceid_queues()
2128 		cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2134 		cs_etm__flush(etmq, tidq);  in cs_etm__clear_all_traceid_queues()
2141 	struct cs_etm_traceid_queue *tidq;  in cs_etm__run_decoder()  local
2143 	tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID);  in cs_etm__run_decoder()
2144 	if (!tidq)  in cs_etm__run_decoder()
2164 			err = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__run_decoder()
2170 			err = cs_etm__end_block(etmq, tidq);  in cs_etm__run_decoder()
2185 		struct cs_etm_traceid_queue *tidq;  in cs_etm__process_timeless_queues()  local
2190 		tidq = cs_etm__etmq_get_traceid_queue(etmq,  in cs_etm__process_timeless_queues()
2193 		if (!tidq)  in cs_etm__process_timeless_queues()
2196 		if ((tid == -1) || (tidq->tid == tid)) {  in cs_etm__process_timeless_queues()
2197 			cs_etm__set_pid_tid_cpu(etm, tidq);  in cs_etm__process_timeless_queues()
2213 	struct cs_etm_traceid_queue *tidq;  in cs_etm__process_queues()  local
2246 		tidq  = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id);  in cs_etm__process_queues()
2247 		if (!tidq) {  in cs_etm__process_queues()
2261 		ret = cs_etm__process_traceid_queue(etmq, tidq);  in cs_etm__process_queues()