1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
24 	int		depth;  member
45 	/* Display overruns? (for self-debug purpose) */
46 	{ TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
48 	{ TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
50 	{ TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
52 	{ TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
54 	{ TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
56 	{ TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
58 	{ TRACER_OPT(funcgraph-irqs, TRACE_GRAPH_PRINT_IRQS) },
60 	{ TRACER_OPT(funcgraph-tail, TRACE_GRAPH_PRINT_TAIL) },
62 	{ TRACER_OPT(sleep-time, TRACE_GRAPH_SLEEP_TIME) },
66 	{ TRACER_OPT(graph-time, TRACE_GRAPH_GRAPH_TIME) },
103 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in __trace_graph_entry()
111 	entry->graph_ent			= *trace;  in __trace_graph_entry()
146 	if (ftrace_graph_notrace_addr(trace->func)) {  in trace_graph_entry()
173 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in trace_graph_entry()
174 	disabled = atomic_inc_return(&data->disabled);  in trace_graph_entry()
182 	atomic_dec(&data->disabled);  in trace_graph_entry()
195 		.depth = 0,  in __trace_graph_function()
199 		.depth    = 0,  in __trace_graph_function()
222 	struct trace_buffer *buffer = tr->array_buffer.buffer;  in __trace_graph_return()
230 	entry->ret				= *trace;  in __trace_graph_return()
253 	data = per_cpu_ptr(tr->array_buffer.data, cpu);  in trace_graph_return()
254 	disabled = atomic_inc_return(&data->disabled);  in trace_graph_return()
259 	atomic_dec(&data->disabled);  in trace_graph_return()
282 	    (trace->rettime - trace->calltime < tracing_thresh))  in trace_graph_thresh_return()
334 	 * Start with a space character - to make it stand out  in print_graph_cpu()
356 	/* 1 stands for the "-" character */  in print_graph_proc()
360 		spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;  in print_graph_proc()
366 	trace_seq_printf(s, "%s-%s", comm, pid_str);  in print_graph_proc()
369 	for (i = 0; i < spaces - (spaces / 2); i++)  in print_graph_proc()
391 	last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);  in verif_pid()
399 	if (prev_pid == -1)  in verif_pid()
402  * Context-switch trace line:  in verif_pid()
404  ------------------------------------------  in verif_pid()
405  | 1)  migration/0--1  =>  sshd-1755  in verif_pid()
406  ------------------------------------------  in verif_pid()
409 	trace_seq_puts(s, " ------------------------------------------\n");  in verif_pid()
414 	trace_seq_puts(s, "\n ------------------------------------------\n\n");  in verif_pid()
421 	struct fgraph_data *data = iter->private;  in get_return_for_leaf()
430 	if (data && data->failed) {  in get_return_for_leaf()
431 		curr = &data->ent;  in get_return_for_leaf()
432 		next = &data->ret;  in get_return_for_leaf()
435 		ring_iter = trace_buffer_iter(iter, iter->cpu);  in get_return_for_leaf()
445 			ring_buffer_consume(iter->array_buffer->buffer, iter->cpu,  in get_return_for_leaf()
447 			event = ring_buffer_peek(iter->array_buffer->buffer, iter->cpu,  in get_return_for_leaf()
461 			data->ent = *curr;  in get_return_for_leaf()
467 			if (next->ent.type == TRACE_GRAPH_RET)  in get_return_for_leaf()
468 				data->ret = *next;  in get_return_for_leaf()
470 				data->ret.ent.type = next->ent.type;  in get_return_for_leaf()
474 	if (next->ent.type != TRACE_GRAPH_RET)  in get_return_for_leaf()
477 	if (curr->ent.pid != next->ent.pid ||  in get_return_for_leaf()
478 			curr->graph_ent.func != next->ret.func)  in get_return_for_leaf()
504 	usecs = iter->ts - iter->array_buffer->time_start;  in print_graph_rel_time()
514 	struct trace_array *tr = iter->tr;  in print_graph_irq()
515 	struct trace_seq *s = &iter->seq;  in print_graph_irq()
516 	struct trace_entry *ent = iter->ent;  in print_graph_irq()
522 	if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {  in print_graph_irq()
525 			print_graph_abs_time(iter->ts, s);  in print_graph_irq()
542 		if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in print_graph_irq()
577 		size_t slen = min_t(size_t, sizeof(nsecs_str), 8UL - len);  in trace_print_graph_duration()
596 	    !(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_duration()
629 	struct fgraph_data *data = iter->private;  in print_graph_entry_leaf()
630 	struct trace_array *tr = iter->tr;  in print_graph_entry_leaf()
634 	int cpu = iter->cpu;  in print_graph_entry_leaf()
637 	graph_ret = &ret_entry->ret;  in print_graph_entry_leaf()
638 	call = &entry->graph_ent;  in print_graph_entry_leaf()
639 	duration = graph_ret->rettime - graph_ret->calltime;  in print_graph_entry_leaf()
644 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_entry_leaf()
647 		 * Comments display at + 1 to depth. Since  in print_graph_entry_leaf()
649 		 * equal to this depth.  in print_graph_entry_leaf()
651 		cpu_data->depth = call->depth - 1;  in print_graph_entry_leaf()
653 		/* No need to keep this function around for this depth */  in print_graph_entry_leaf()
654 		if (call->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_entry_leaf()
655 		    !WARN_ON_ONCE(call->depth < 0))  in print_graph_entry_leaf()
656 			cpu_data->enter_funcs[call->depth] = 0;  in print_graph_entry_leaf()
663 	for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_entry_leaf()
666 	trace_seq_printf(s, "%ps();\n", (void *)call->func);  in print_graph_entry_leaf()
668 	print_graph_irq(iter, graph_ret->func, TRACE_GRAPH_RET,  in print_graph_entry_leaf()
669 			cpu, iter->ent->pid, flags);  in print_graph_entry_leaf()
679 	struct ftrace_graph_ent *call = &entry->graph_ent;  in print_graph_entry_nested()
680 	struct fgraph_data *data = iter->private;  in print_graph_entry_nested()
681 	struct trace_array *tr = iter->tr;  in print_graph_entry_nested()
686 		int cpu = iter->cpu;  in print_graph_entry_nested()
688 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_entry_nested()
689 		cpu_data->depth = call->depth;  in print_graph_entry_nested()
692 		if (call->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_entry_nested()
693 		    !WARN_ON_ONCE(call->depth < 0))  in print_graph_entry_nested()
694 			cpu_data->enter_funcs[call->depth] = call->func;  in print_graph_entry_nested()
701 	for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_entry_nested()
704 	trace_seq_printf(s, "%ps() {\n", (void *)call->func);  in print_graph_entry_nested()
720 	struct fgraph_data *data = iter->private;  in print_graph_prologue()
721 	struct trace_entry *ent = iter->ent;  in print_graph_prologue()
722 	struct trace_array *tr = iter->tr;  in print_graph_prologue()
723 	int cpu = iter->cpu;  in print_graph_prologue()
726 	verif_pid(s, ent->pid, cpu, data);  in print_graph_prologue()
730 		print_graph_irq(iter, addr, type, cpu, ent->pid, flags);  in print_graph_prologue()
732 	if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_prologue()
737 		print_graph_abs_time(iter->ts, s);  in print_graph_prologue()
749 		print_graph_proc(s, ent->pid);  in print_graph_prologue()
754 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)  in print_graph_prologue()
764  *  - we are inside irq code
765  *  - we just entered irq code
768  *  - funcgraph-interrupts option is set
769  *  - we are not inside irq code
773 		unsigned long addr, int depth)  in check_irq_entry()  argument
775 	int cpu = iter->cpu;  in check_irq_entry()
777 	struct fgraph_data *data = iter->private;  in check_irq_entry()
788 	depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in check_irq_entry()
803 	*depth_irq = depth;  in check_irq_entry()
811  *  - we are inside irq code
812  *  - we just left irq code
815  *  - funcgraph-interrupts option is set
816  *  - we are not inside irq code
819 check_irq_return(struct trace_iterator *iter, u32 flags, int depth)  in check_irq_return()  argument
821 	int cpu = iter->cpu;  in check_irq_return()
823 	struct fgraph_data *data = iter->private;  in check_irq_return()
834 	depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in check_irq_return()
839 	if (*depth_irq == -1)  in check_irq_return()
844 	 * Let's not trace it and clear the entry depth, since  in check_irq_return()
848 	 * we are out of the entry depth. Thus protecting us from  in check_irq_return()
851 	if (*depth_irq >= depth) {  in check_irq_return()
852 		*depth_irq = -1;  in check_irq_return()
866 	struct fgraph_data *data = iter->private;  in print_graph_entry()
867 	struct ftrace_graph_ent *call = &field->graph_ent;  in print_graph_entry()
870 	int cpu = iter->cpu;  in print_graph_entry()
872 	if (check_irq_entry(iter, flags, call->func, call->depth))  in print_graph_entry()
875 	print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func, flags);  in print_graph_entry()
888 		if (s->full) {  in print_graph_entry()
889 			data->failed = 1;  in print_graph_entry()
890 			data->cpu = cpu;  in print_graph_entry()
892 			data->failed = 0;  in print_graph_entry()
903 	unsigned long long duration = trace->rettime - trace->calltime;  in print_graph_return()
904 	struct fgraph_data *data = iter->private;  in print_graph_return()
905 	struct trace_array *tr = iter->tr;  in print_graph_return()
906 	pid_t pid = ent->pid;  in print_graph_return()
907 	int cpu = iter->cpu;  in print_graph_return()
911 	if (check_irq_return(iter, flags, trace->depth))  in print_graph_return()
916 		int cpu = iter->cpu;  in print_graph_return()
918 		cpu_data = per_cpu_ptr(data->cpu_data, cpu);  in print_graph_return()
921 		 * Comments display at + 1 to depth. This is the  in print_graph_return()
925 		cpu_data->depth = trace->depth - 1;  in print_graph_return()
927 		if (trace->depth < FTRACE_RETFUNC_DEPTH &&  in print_graph_return()
928 		    !WARN_ON_ONCE(trace->depth < 0)) {  in print_graph_return()
929 			if (cpu_data->enter_funcs[trace->depth] != trace->func)  in print_graph_return()
931 			cpu_data->enter_funcs[trace->depth] = 0;  in print_graph_return()
941 	for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++)  in print_graph_return()
949 	 * that if the funcgraph-tail option is enabled.  in print_graph_return()
954 		trace_seq_printf(s, "} /* %ps */\n", (void *)trace->func);  in print_graph_return()
959 				 trace->overrun);  in print_graph_return()
961 	print_graph_irq(iter, trace->func, TRACE_GRAPH_RET,  in print_graph_return()
971 	struct trace_array *tr = iter->tr;  in print_graph_comment()
972 	unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);  in print_graph_comment()
973 	struct fgraph_data *data = iter->private;  in print_graph_comment()
975 	int depth = 0;  in print_graph_comment()  local
980 		depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth;  in print_graph_comment()
988 	if (depth > 0)  in print_graph_comment()
989 		for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++)  in print_graph_comment()
995 	switch (iter->ent->type) {  in print_graph_comment()
1012 		event = ftrace_find_event(ent->type);  in print_graph_comment()
1016 		ret = event->funcs->trace(iter, sym_flags, event);  in print_graph_comment()
1025 	if (s->buffer[s->seq.len - 1] == '\n') {  in print_graph_comment()
1026 		s->buffer[s->seq.len - 1] = '\0';  in print_graph_comment()
1027 		s->seq.len--;  in print_graph_comment()
1040 	struct fgraph_data *data = iter->private;  in print_graph_function_flags()
1041 	struct trace_entry *entry = iter->ent;  in print_graph_function_flags()
1042 	struct trace_seq *s = &iter->seq;  in print_graph_function_flags()
1043 	int cpu = iter->cpu;  in print_graph_function_flags()
1046 	if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) {  in print_graph_function_flags()
1047 		per_cpu_ptr(data->cpu_data, cpu)->ignore = 0;  in print_graph_function_flags()
1055 	if (data && data->failed) {  in print_graph_function_flags()
1056 		field = &data->ent;  in print_graph_function_flags()
1057 		iter->cpu = data->cpu;  in print_graph_function_flags()
1059 		if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) {  in print_graph_function_flags()
1060 			per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1;  in print_graph_function_flags()
1063 		iter->cpu = cpu;  in print_graph_function_flags()
1067 	switch (entry->type) {  in print_graph_function_flags()
1083 		return print_graph_return(&field->ret, s, entry, iter, flags);  in print_graph_function_flags()
1126 	seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);  in print_lat_header()
1127 	seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);  in print_lat_header()
1128 	seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);  in print_lat_header()
1129 	seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);  in print_lat_header()
1136 	int lat = tr->trace_flags & TRACE_ITER_LATENCY_FMT;  in __print_graph_headers_flags()
1181 	struct trace_iterator *iter = s->private;  in print_graph_headers_flags()
1182 	struct trace_array *tr = iter->tr;  in print_graph_headers_flags()
1184 	if (!(tr->trace_flags & TRACE_ITER_CONTEXT_INFO))  in print_graph_headers_flags()
1187 	if (tr->trace_flags & TRACE_ITER_LATENCY_FMT) {  in print_graph_headers_flags()
1200 	/* pid and depth on the last trace processed */  in graph_trace_open()
1205 	iter->private = NULL;  in graph_trace_open()
1214 	data->cpu_data = alloc_percpu_gfp(struct fgraph_cpu_data, gfpflags);  in graph_trace_open()
1215 	if (!data->cpu_data)  in graph_trace_open()
1219 		pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);  in graph_trace_open()
1220 		int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);  in graph_trace_open()  local
1221 		int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore);  in graph_trace_open()
1222 		int *depth_irq = &(per_cpu_ptr(data->cpu_data, cpu)->depth_irq);  in graph_trace_open()
1224 		*pid = -1;  in graph_trace_open()
1225 		*depth = 0;  in graph_trace_open()
1227 		*depth_irq = -1;  in graph_trace_open()
1230 	iter->private = data;  in graph_trace_open()
1242 	struct fgraph_data *data = iter->private;  in graph_trace_close()
1245 		free_percpu(data->cpu_data);  in graph_trace_close()
1352 	max_bytes_for_cpu = snprintf(NULL, 0, "%u", nr_cpu_ids - 1);  in init_graph_trace()