74 static int thread_stack__grow(struct thread_stack *ts)  in thread_stack__grow()  argument
79 	new_sz = ts->sz + STACK_GROWTH;  in thread_stack__grow()
82 	new_stack = realloc(ts->stack, sz);  in thread_stack__grow()
86 	ts->stack = new_stack;  in thread_stack__grow()
87 	ts->sz = new_sz;  in thread_stack__grow()
95 	struct thread_stack *ts;  in thread_stack__new()  local
97 	ts = zalloc(sizeof(struct thread_stack));  in thread_stack__new()
98 	if (!ts)  in thread_stack__new()
101 	if (thread_stack__grow(ts)) {  in thread_stack__new()
102 		free(ts);  in thread_stack__new()
107 		ts->kernel_start = machine__kernel_start(thread->mg->machine);  in thread_stack__new()
109 		ts->kernel_start = 1ULL << 63;  in thread_stack__new()
110 	ts->crp = crp;  in thread_stack__new()
112 	return ts;  in thread_stack__new()
115 static int thread_stack__push(struct thread_stack *ts, u64 ret_addr)  in thread_stack__push()  argument
119 	if (ts->cnt == ts->sz) {  in thread_stack__push()
120 		err = thread_stack__grow(ts);  in thread_stack__push()
123 			ts->cnt = 0;  in thread_stack__push()
127 	ts->stack[ts->cnt++].ret_addr = ret_addr;  in thread_stack__push()
132 static void thread_stack__pop(struct thread_stack *ts, u64 ret_addr)  in thread_stack__pop()  argument
145 	for (i = ts->cnt; i; ) {  in thread_stack__pop()
146 		if (ts->stack[--i].ret_addr == ret_addr) {  in thread_stack__pop()
147 			ts->cnt = i;  in thread_stack__pop()
153 static bool thread_stack__in_kernel(struct thread_stack *ts)  in thread_stack__in_kernel()  argument
155 	if (!ts->cnt)  in thread_stack__in_kernel()
158 	return ts->stack[ts->cnt - 1].cp->in_kernel;  in thread_stack__in_kernel()
162 				     struct thread_stack *ts, size_t idx,  in thread_stack__call_return()  argument
165 	struct call_return_processor *crp = ts->crp;  in thread_stack__call_return()
169 		.comm = ts->comm,  in thread_stack__call_return()
173 	tse = &ts->stack[idx];  in thread_stack__call_return()
177 	cr.branch_count = ts->branch_count - tse->branch_count;  in thread_stack__call_return()
188 static int __thread_stack__flush(struct thread *thread, struct thread_stack *ts)  in __thread_stack__flush()  argument
190 	struct call_return_processor *crp = ts->crp;  in __thread_stack__flush()
194 		ts->cnt = 0;  in __thread_stack__flush()
198 	while (ts->cnt) {  in __thread_stack__flush()
199 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in __thread_stack__flush()
200 						ts->last_time, 0, true);  in __thread_stack__flush()
203 			ts->cnt = 0;  in __thread_stack__flush()
213 	if (thread->ts)  in thread_stack__flush()
214 		return __thread_stack__flush(thread, thread->ts);  in thread_stack__flush()
225 	if (!thread->ts) {  in thread_stack__event()
226 		thread->ts = thread_stack__new(thread, NULL);  in thread_stack__event()
227 		if (!thread->ts) {  in thread_stack__event()
231 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
239 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__event()
240 		if (thread->ts->trace_nr)  in thread_stack__event()
241 			__thread_stack__flush(thread, thread->ts);  in thread_stack__event()
242 		thread->ts->trace_nr = trace_nr;  in thread_stack__event()
246 	if (thread->ts->crp)  in thread_stack__event()
257 		return thread_stack__push(thread->ts, ret_addr);  in thread_stack__event()
261 		thread_stack__pop(thread->ts, to_ip);  in thread_stack__event()
269 	if (!thread || !thread->ts)  in thread_stack__set_trace_nr()
272 	if (trace_nr != thread->ts->trace_nr) {  in thread_stack__set_trace_nr()
273 		if (thread->ts->trace_nr)  in thread_stack__set_trace_nr()
274 			__thread_stack__flush(thread, thread->ts);  in thread_stack__set_trace_nr()
275 		thread->ts->trace_nr = trace_nr;  in thread_stack__set_trace_nr()
281 	if (thread->ts) {  in thread_stack__free()
282 		__thread_stack__flush(thread, thread->ts);  in thread_stack__free()
283 		zfree(&thread->ts->stack);  in thread_stack__free()
284 		zfree(&thread->ts);  in thread_stack__free()
293 	if (!thread || !thread->ts)  in thread_stack__sample()
296 		chain->nr = min(sz, thread->ts->cnt + 1);  in thread_stack__sample()
301 		chain->ips[i] = thread->ts->stack[thread->ts->cnt - i].ret_addr;  in thread_stack__sample()
333 static int thread_stack__push_cp(struct thread_stack *ts, u64 ret_addr,  in thread_stack__push_cp()  argument
340 	if (ts->cnt == ts->sz) {  in thread_stack__push_cp()
341 		err = thread_stack__grow(ts);  in thread_stack__push_cp()
346 	tse = &ts->stack[ts->cnt++];  in thread_stack__push_cp()
350 	tse->branch_count = ts->branch_count;  in thread_stack__push_cp()
357 static int thread_stack__pop_cp(struct thread *thread, struct thread_stack *ts,  in thread_stack__pop_cp()  argument
363 	if (!ts->cnt)  in thread_stack__pop_cp()
366 	if (ts->cnt == 1) {  in thread_stack__pop_cp()
367 		struct thread_stack_entry *tse = &ts->stack[0];  in thread_stack__pop_cp()
370 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
374 	if (ts->stack[ts->cnt - 1].ret_addr == ret_addr) {  in thread_stack__pop_cp()
375 		return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
378 		size_t i = ts->cnt - 1;  in thread_stack__pop_cp()
381 			if (ts->stack[i].ret_addr != ret_addr)  in thread_stack__pop_cp()
384 			while (ts->cnt > i) {  in thread_stack__pop_cp()
385 				err = thread_stack__call_return(thread, ts,  in thread_stack__pop_cp()
386 								--ts->cnt,  in thread_stack__pop_cp()
392 			return thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__pop_cp()
400 static int thread_stack__bottom(struct thread *thread, struct thread_stack *ts,  in thread_stack__bottom()  argument
405 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__bottom()
421 				ts->kernel_start);  in thread_stack__bottom()
425 	return thread_stack__push_cp(thread->ts, ip, sample->time, ref, cp,  in thread_stack__bottom()
430 					struct thread_stack *ts,  in thread_stack__no_call_return()  argument
435 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__no_call_return()
437 	u64 ks = ts->kernel_start;  in thread_stack__no_call_return()
442 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
443 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
451 		if (!ts->cnt) {  in thread_stack__no_call_return()
454 						ts->kernel_start);  in thread_stack__no_call_return()
457 			return thread_stack__push_cp(ts, 0, sample->time, ref,  in thread_stack__no_call_return()
460 	} else if (thread_stack__in_kernel(ts) && sample->ip < ks) {  in thread_stack__no_call_return()
462 		while (thread_stack__in_kernel(ts)) {  in thread_stack__no_call_return()
463 			err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__no_call_return()
471 	if (ts->cnt)  in thread_stack__no_call_return()
472 		parent = ts->stack[ts->cnt - 1].cp;  in thread_stack__no_call_return()
478 				ts->kernel_start);  in thread_stack__no_call_return()
482 	err = thread_stack__push_cp(ts, sample->addr, sample->time, ref, cp,  in thread_stack__no_call_return()
487 	return thread_stack__pop_cp(thread, ts, sample->addr, sample->time, ref,  in thread_stack__no_call_return()
492 				     struct thread_stack *ts, u64 timestamp,  in thread_stack__trace_begin()  argument
498 	if (!ts->cnt)  in thread_stack__trace_begin()
502 	tse = &ts->stack[ts->cnt - 1];  in thread_stack__trace_begin()
504 		err = thread_stack__call_return(thread, ts, --ts->cnt,  in thread_stack__trace_begin()
513 static int thread_stack__trace_end(struct thread_stack *ts,  in thread_stack__trace_end()  argument
516 	struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__trace_end()
521 	if (!ts->cnt || (ts->cnt == 1 && ts->stack[0].ref == ref))  in thread_stack__trace_end()
524 	cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp, NULL, 0,  in thread_stack__trace_end()
525 				ts->kernel_start);  in thread_stack__trace_end()
531 	return thread_stack__push_cp(ts, ret_addr, sample->time, ref, cp,  in thread_stack__trace_end()
541 	struct thread_stack *ts = thread->ts;  in thread_stack__process()  local
544 	if (ts) {  in thread_stack__process()
545 		if (!ts->crp) {  in thread_stack__process()
548 			thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
549 			if (!thread->ts)  in thread_stack__process()
551 			ts = thread->ts;  in thread_stack__process()
552 			ts->comm = comm;  in thread_stack__process()
555 		thread->ts = thread_stack__new(thread, crp);  in thread_stack__process()
556 		if (!thread->ts)  in thread_stack__process()
558 		ts = thread->ts;  in thread_stack__process()
559 		ts->comm = comm;  in thread_stack__process()
563 	if (ts->comm != comm && thread->pid_ == thread->tid) {  in thread_stack__process()
564 		err = __thread_stack__flush(thread, ts);  in thread_stack__process()
567 		ts->comm = comm;  in thread_stack__process()
571 	if (!ts->cnt) {  in thread_stack__process()
572 		err = thread_stack__bottom(thread, ts, sample, from_al, to_al,  in thread_stack__process()
578 	ts->branch_count += 1;  in thread_stack__process()
579 	ts->last_time = sample->time;  in thread_stack__process()
582 		struct call_path_root *cpr = ts->crp->cpr;  in thread_stack__process()
593 		cp = call_path__findnew(cpr, ts->stack[ts->cnt - 1].cp,  in thread_stack__process()
595 					ts->kernel_start);  in thread_stack__process()
598 		err = thread_stack__push_cp(ts, ret_addr, sample->time, ref,  in thread_stack__process()
604 		err = thread_stack__pop_cp(thread, ts, sample->addr,  in thread_stack__process()
609 			err = thread_stack__no_call_return(thread, ts, sample,  in thread_stack__process()
613 		err = thread_stack__trace_begin(thread, ts, sample->time, ref);  in thread_stack__process()
615 		err = thread_stack__trace_end(ts, sample, ref);  in thread_stack__process()
623 	if (!thread->ts)  in thread_stack__depth()
625 	return thread->ts->cnt;  in thread_stack__depth()