110 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
115 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
118 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
122 	ts->stack = new_stack;  in thread_stack__grow()
123 	ts->sz = new_sz;  in thread_stack__grow()
128 static int thread_stack__init(struct thread_stack *ts, struct thread *thread,  in thread_stack__init()  argument
133 	err = thread_stack__grow(ts);  in thread_stack__init()
141 		ts->kernel_start = machine__kernel_start(machine);  in thread_stack__init()
143 			ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__init()
145 		ts->kernel_start = 1ULL << 63;  in thread_stack__init()
147 	ts->crp = crp;  in thread_stack__init()
155 	struct thread_stack *ts = thread->ts, *new_ts;  in thread_stack__new()  local
156 	unsigned int old_sz = ts ? ts->arr_sz : 0;  in thread_stack__new()
162 	if (!ts || new_sz > old_sz) {  in thread_stack__new()
163 		new_ts = calloc(new_sz, sizeof(*ts));  in thread_stack__new()
166 		if (ts)  in thread_stack__new()
167 			memcpy(new_ts, ts, old_sz * sizeof(*ts));  in thread_stack__new()
169 		zfree(&thread->ts);  in thread_stack__new()
170 		thread->ts = new_ts;  in thread_stack__new()
171 		ts = new_ts;  in thread_stack__new()
175 	    (unsigned int)cpu < ts->arr_sz)  in thread_stack__new()
176 		ts += cpu;  in thread_stack__new()
178 	if (!ts->stack &&  in thread_stack__new()
179 	    thread_stack__init(ts, thread, crp))  in thread_stack__new()
182 	return ts;  in thread_stack__new()
187 	struct thread_stack *ts = thread->ts;  in thread__cpu_stack()  local
192 	if (!ts || (unsigned int)cpu >= ts->arr_sz)  in thread__cpu_stack()
195 	ts += cpu;  in thread__cpu_stack()
197 	if (!ts->stack)  in thread__cpu_stack()
200 	return ts;  in thread__cpu_stack()
212 	return thread->ts;  in thread__stack()
215 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push()  argument
220 	if (ts->cnt == ts->sz) {  in thread_stack__push()
221 		err = thread_stack__grow(ts);  in thread_stack__push()
224 			ts->cnt = 0;  in thread_stack__push()
228 	ts->stack[ts->cnt].trace_end = trace_end;  in thread_stack__push()
229 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
234 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
247 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
248 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
249 			ts->cnt = i;  in thread_stack__pop()
255 static void thread_stack__pop_trace_end(struct thread_stack *ts)  in thread_stack__pop_trace_end()  argument
259 	for (i = ts->cnt; i; ) {  in thread_stack__pop_trace_end()
260 		if (ts->stack[--i].trace_end)  in thread_stack__pop_trace_end()
261 			ts->cnt = i;  in thread_stack__pop_trace_end()
267 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
269 	if (!ts->cnt)  in thread_stack__in_kernel()
272 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
276 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
279 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
283 		.comm = ts->comm,  in thread_stack__call_return()
288 	tse = &ts->stack[idx];  in thread_stack__call_return()
292 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
293 	cr.insn_count = ts->insn_count - tse->insn_count;  in thread_stack__call_return()
294 	cr.cyc_count = ts->cyc_count - tse->cyc_count;  in thread_stack__call_return()
315 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in __thread_stack__flush()  argument
317 	struct call_return_processor *crp = ts->crp;  in __thread_stack__flush()
321 		ts->cnt = 0;  in __thread_stack__flush()
325 	while (ts->cnt) {  in __thread_stack__flush()
326 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in __thread_stack__flush()
327 						ts->last_time, 0, true);  in __thread_stack__flush()
330 			ts->cnt = 0;  in __thread_stack__flush()
340 	struct thread_stack *ts = thread->ts;  in thread_stack__flush()  local
344 	if (ts) {  in thread_stack__flush()
345 		for (pos = 0; pos < ts->arr_sz; pos++) {  in thread_stack__flush()
346 			int ret = __thread_stack__flush(thread, ts + pos);  in thread_stack__flush()
359 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__event()  local
364 	if (!ts) {  in thread_stack__event()
365 		ts = thread_stack__new(thread, cpu, NULL);  in thread_stack__event()
366 		if (!ts) {  in thread_stack__event()
370 		ts->trace_nr = trace_nr;  in thread_stack__event()
378 	if (trace_nr != ts->trace_nr) {  in thread_stack__event()
379 		if (ts->trace_nr)  in thread_stack__event()
380 			__thread_stack__flush(thread, ts);  in thread_stack__event()
381 		ts->trace_nr = trace_nr;  in thread_stack__event()
385 	if (ts->crp)  in thread_stack__event()
396 		return thread_stack__push(ts, ret_addr,  in thread_stack__event()
406 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
407 		thread_stack__pop_trace_end(ts);  in thread_stack__event()
409 		thread_stack__pop(ts, to_ip);  in thread_stack__event()
417 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__set_trace_nr()  local
419 	if (!ts)  in thread_stack__set_trace_nr()
422 	if (trace_nr != ts->trace_nr) {  in thread_stack__set_trace_nr()
423 		if (ts->trace_nr)  in thread_stack__set_trace_nr()
424 			__thread_stack__flush(thread, ts);  in thread_stack__set_trace_nr()
425 		ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
429 static void __thread_stack__free(struct thread *thread, struct thread_stack *ts)  in __thread_stack__free()  argument
431 	__thread_stack__flush(thread, ts);  in __thread_stack__free()
432 	zfree(&ts->stack);  in __thread_stack__free()
435 static void thread_stack__reset(struct thread *thread, struct thread_stack *ts)  in thread_stack__reset()  argument
437 	unsigned int arr_sz = ts->arr_sz;  in thread_stack__reset()
439 	__thread_stack__free(thread, ts);  in thread_stack__reset()
440 	memset(ts, 0, sizeof(*ts));  in thread_stack__reset()
441 	ts->arr_sz = arr_sz;  in thread_stack__reset()
446 	struct thread_stack *ts = thread->ts;  in thread_stack__free()  local
449 	if (ts) {  in thread_stack__free()
450 		for (pos = 0; pos < ts->arr_sz; pos++)  in thread_stack__free()
451 			__thread_stack__free(thread, ts + pos);  in thread_stack__free()
452 		zfree(&thread->ts);  in thread_stack__free()
465 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__sample()  local
478 	if (!ts) {  in thread_stack__sample()
485 	for (i = 2, j = 1; i < sz && j <= ts->cnt; i++, j++) {  in thread_stack__sample()
486 		ip = ts->stack[ts->cnt - j].ret_addr;  in thread_stack__sample()
529 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
539 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
540 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
545 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
549 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
550 	tse->insn_count = ts->insn_count;  in thread_stack__push_cp()
551 	tse->cyc_count = ts->cyc_count;  in thread_stack__push_cp()
561 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
567 	if (!ts->cnt)  in thread_stack__pop_cp()
570 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
571 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
574 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
578 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr &&  in thread_stack__pop_cp()
579 	    !ts->stack[ts->cnt - 1].non_call) {  in thread_stack__pop_cp()
580 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
583 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
586 			if (ts->stack[i].ret_addr != ret_addr ||  in thread_stack__pop_cp()
587 			    ts->stack[i].non_call)  in thread_stack__pop_cp()
590 			while (ts->cnt > i) {  in thread_stack__pop_cp()
591 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
592 								--ts->cnt,  in thread_stack__pop_cp()
598 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
606 static int thread_stack__bottom(struct thread_stack *ts,  in thread_stack__bottom()  argument
611 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
627 				ts->kernel_start);  in thread_stack__bottom()
629 	return thread_stack__push_cp(ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
633 static int thread_stack__pop_ks(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_ks()  argument
640 	while (thread_stack__in_kernel(ts)) {  in thread_stack__pop_ks()
641 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_ks()
651 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
656 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
661 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
669 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
674 		if (!ts->cnt) {  in thread_stack__no_call_return()
676 			return thread_stack__push_cp(ts, 0, tm, ref, cp, true,  in thread_stack__no_call_return()
679 	} else if (thread_stack__in_kernel(ts) && ip < ks) {  in thread_stack__no_call_return()
681 		err = thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__no_call_return()
686 	if (ts->cnt)  in thread_stack__no_call_return()
687 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
697 		if (ts->cnt == 1) {  in thread_stack__no_call_return()
698 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
704 		if (!ts->cnt) {  in thread_stack__no_call_return()
707 			return thread_stack__push_cp(ts, addr, tm, ref, cp,  in thread_stack__no_call_return()
717 		err = thread_stack__push_cp(ts, 0, tm, ref, cp, true, false);  in thread_stack__no_call_return()
719 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__no_call_return()
731 	err = thread_stack__push_cp(ts, addr, tm, ref, cp, true, false);  in thread_stack__no_call_return()
737 	err = thread_stack__push_cp(ts, ip, tm, ref, cp, true, false);  in thread_stack__no_call_return()
741 	return thread_stack__call_return(thread, ts, --ts->cnt, tm, ref, false);  in thread_stack__no_call_return()
745 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
751 	if (!ts->cnt)  in thread_stack__trace_begin()
755 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
757 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
766 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
769 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
774 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
777 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
778 				ts->kernel_start);  in thread_stack__trace_end()
782 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
798 static int thread_stack__x86_retpoline(struct thread_stack *ts,  in thread_stack__x86_retpoline()  argument
802 	struct thread_stack_entry *tse = &ts->stack[ts->cnt - 1];  in thread_stack__x86_retpoline()
803 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__x86_retpoline()
819 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
820 		sym = ts->stack[ts->cnt - 2].cp->sym;  in thread_stack__x86_retpoline()
827 			ts->cnt -= 1;  in thread_stack__x86_retpoline()
835 		ts->cnt -= 1;  in thread_stack__x86_retpoline()
839 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 2].cp, tsym,  in thread_stack__x86_retpoline()
840 				sample->addr, ts->kernel_start);  in thread_stack__x86_retpoline()
845 	ts->stack[ts->cnt - 1].cp = cp;  in thread_stack__x86_retpoline()
856 	struct thread_stack *ts = thread__stack(thread, sample->cpu);  in thread_stack__process()  local
860 	if (ts && !ts->crp) {  in thread_stack__process()
862 		thread_stack__reset(thread, ts);  in thread_stack__process()
863 		ts = NULL;  in thread_stack__process()
866 	if (!ts) {  in thread_stack__process()
867 		ts = thread_stack__new(thread, sample->cpu, crp);  in thread_stack__process()
868 		if (!ts)  in thread_stack__process()
870 		ts->comm = comm;  in thread_stack__process()
873 	rstate = ts->rstate;  in thread_stack__process()
875 		ts->rstate = X86_RETPOLINE_POSSIBLE;  in thread_stack__process()
878 	if (ts->comm != comm && thread->pid_ == thread->tid) {  in thread_stack__process()
879 		err = __thread_stack__flush(thread, ts);  in thread_stack__process()
882 		ts->comm = comm;  in thread_stack__process()
886 	if (!ts->cnt) {  in thread_stack__process()
887 		err = thread_stack__bottom(ts, sample, from_al, to_al, ref);  in thread_stack__process()
892 	ts->branch_count += 1;  in thread_stack__process()
893 	ts->insn_count += sample->insn_cnt;  in thread_stack__process()
894 	ts->cyc_count += sample->cyc_cnt;  in thread_stack__process()
895 	ts->last_time = sample->time;  in thread_stack__process()
899 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
910 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
912 					ts->kernel_start);  in thread_stack__process()
913 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
923 			ts->rstate = X86_RETPOLINE_DETECTED;  in thread_stack__process()
934 			return thread_stack__pop_ks(thread, ts, sample, ref);  in thread_stack__process()
941 		if (rstate == X86_RETPOLINE_DETECTED && ts->cnt > 2 &&  in thread_stack__process()
942 		    ts->stack[ts->cnt - 1].ret_addr != sample->addr)  in thread_stack__process()
943 			return thread_stack__x86_retpoline(ts, sample, to_al);  in thread_stack__process()
945 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
950 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
954 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
956 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()
960 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
969 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
971 					ts->kernel_start);  in thread_stack__process()
972 		err = thread_stack__push_cp(ts, 0, sample->time, ref, cp, false,  in thread_stack__process()
975 			ts->stack[ts->cnt - 1].non_call = true;  in thread_stack__process()
983 	struct thread_stack *ts = thread__stack(thread, cpu);  in thread_stack__depth()  local
985 	if (!ts)  in thread_stack__depth()
987 	return ts->cnt;  in thread_stack__depth()