14 unsigned long unwind_get_return_address(struct unwind_state *state)  in unwind_get_return_address()  argument
16 	if (unwind_done(state))  in unwind_get_return_address()
19 	return __kernel_text_address(state->ip) ? state->ip : 0;  in unwind_get_return_address()
23 unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)  in unwind_get_return_address_ptr()  argument
25 	if (unwind_done(state))  in unwind_get_return_address_ptr()
28 	return state->regs ? &state->regs->ip : state->bp + 1;  in unwind_get_return_address_ptr()
31 static void unwind_dump(struct unwind_state *state)  in unwind_dump()  argument
45 			state->stack_info.type, state->stack_info.next_sp,  in unwind_dump()
46 			state->stack_mask, state->graph_idx);  in unwind_dump()
48 	for (sp = PTR_ALIGN(state->orig_sp, sizeof(long)); sp;  in unwind_dump()
50 		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))  in unwind_dump()
86 static inline unsigned long *last_frame(struct unwind_state *state)  in last_frame()  argument
88 	return (unsigned long *)task_pt_regs(state->task) - 2;  in last_frame()
91 static bool is_last_frame(struct unwind_state *state)  in is_last_frame()  argument
93 	return state->bp == last_frame(state);  in is_last_frame()
102 static inline unsigned long *last_aligned_frame(struct unwind_state *state)  in last_aligned_frame()  argument
104 	return last_frame(state) - GCC_REALIGN_WORDS;  in last_aligned_frame()
107 static bool is_last_aligned_frame(struct unwind_state *state)  in is_last_aligned_frame()  argument
109 	unsigned long *last_bp = last_frame(state);  in is_last_aligned_frame()
110 	unsigned long *aligned_bp = last_aligned_frame(state);  in is_last_aligned_frame()
135 	return (state->bp == aligned_bp && *(aligned_bp + 1) == *(last_bp + 1));  in is_last_aligned_frame()
138 static bool is_last_ftrace_frame(struct unwind_state *state)  in is_last_ftrace_frame()  argument
140 	unsigned long *last_bp = last_frame(state);  in is_last_ftrace_frame()
155 	return (state->bp == last_ftrace_bp &&  in is_last_ftrace_frame()
156 		*state->bp == *(state->bp + 2) &&  in is_last_ftrace_frame()
157 		*(state->bp + 1) == *(state->bp + 4));  in is_last_ftrace_frame()
160 static bool is_last_task_frame(struct unwind_state *state)  in is_last_task_frame()  argument
162 	return is_last_frame(state) || is_last_aligned_frame(state) ||  in is_last_task_frame()
163 	       is_last_ftrace_frame(state);  in is_last_task_frame()
192 static bool update_stack_state(struct unwind_state *state,  in update_stack_state()  argument
195 	struct stack_info *info = &state->stack_info;  in update_stack_state()
201 	if (state->regs)  in update_stack_state()
202 		prev_frame_end = (void *)state->regs + sizeof(*state->regs);  in update_stack_state()
204 		prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;  in update_stack_state()
211 		state->got_irq = true;  in update_stack_state()
225 		if (get_stack_info(info->next_sp, state->task, info,  in update_stack_state()
226 				   &state->stack_mask))  in update_stack_state()
230 	if (state->orig_sp && state->stack_info.type == prev_type &&  in update_stack_state()
234 	/* Move state to the next frame: */  in update_stack_state()
236 		state->regs = regs;  in update_stack_state()
237 		state->bp = NULL;  in update_stack_state()
239 		state->bp = next_bp;  in update_stack_state()
240 		state->regs = NULL;  in update_stack_state()
244 	if (state->regs && user_mode(state->regs))  in update_stack_state()
245 		state->ip = 0;  in update_stack_state()
247 		addr_p = unwind_get_return_address_ptr(state);  in update_stack_state()
248 		addr = READ_ONCE_TASK_STACK(state->task, *addr_p);  in update_stack_state()
249 		state->ip = ftrace_graph_ret_addr(state->task, &state->graph_idx,  in update_stack_state()
254 	if (!state->orig_sp)  in update_stack_state()
255 		state->orig_sp = frame;  in update_stack_state()
260 bool unwind_next_frame(struct unwind_state *state)  in unwind_next_frame()  argument
265 	if (unwind_done(state))  in unwind_next_frame()
269 	if (state->regs && user_mode(state->regs))  in unwind_next_frame()
272 	if (is_last_task_frame(state)) {  in unwind_next_frame()
273 		regs = task_pt_regs(state->task);  in unwind_next_frame()
294 		state->regs = regs;  in unwind_next_frame()
295 		state->bp = NULL;  in unwind_next_frame()
296 		state->ip = 0;  in unwind_next_frame()
301 	if (state->next_bp) {  in unwind_next_frame()
302 		next_bp = state->next_bp;  in unwind_next_frame()
303 		state->next_bp = NULL;  in unwind_next_frame()
304 	} else if (state->regs) {  in unwind_next_frame()
305 		next_bp = (unsigned long *)state->regs->bp;  in unwind_next_frame()
307 		next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp);  in unwind_next_frame()
311 	if (!update_stack_state(state, next_bp))  in unwind_next_frame()
317 	state->error = true;  in unwind_next_frame()
326 	if (state->task != current)  in unwind_next_frame()
333 	if (state->got_irq && in_entry_code(state->ip))  in unwind_next_frame()
335 	if (state->regs &&  in unwind_next_frame()
336 	    state->regs->sp >= (unsigned long)last_aligned_frame(state) &&  in unwind_next_frame()
337 	    state->regs->sp < (unsigned long)task_pt_regs(state->task))  in unwind_next_frame()
347 	if (state->regs) {  in unwind_next_frame()
350 			state->regs, state->task->comm,  in unwind_next_frame()
351 			state->task->pid, next_bp);  in unwind_next_frame()
352 		unwind_dump(state);  in unwind_next_frame()
356 			state->bp, state->task->comm,  in unwind_next_frame()
357 			state->task->pid, next_bp);  in unwind_next_frame()
358 		unwind_dump(state);  in unwind_next_frame()
361 	state->stack_info.type = STACK_TYPE_UNKNOWN;  in unwind_next_frame()
366 void __unwind_start(struct unwind_state *state, struct task_struct *task,  in __unwind_start()  argument
371 	memset(state, 0, sizeof(*state));  in __unwind_start()
372 	state->task = task;  in __unwind_start()
373 	state->got_irq = (regs);  in __unwind_start()
377 		state->stack_info.type = STACK_TYPE_UNKNOWN;  in __unwind_start()
393 		state->next_bp = bp;  in __unwind_start()
398 	get_stack_info(bp, state->task, &state->stack_info,  in __unwind_start()
399 		       &state->stack_mask);  in __unwind_start()
400 	update_stack_state(state, bp);  in __unwind_start()
407 	while (!unwind_done(state) &&  in __unwind_start()
408 	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||  in __unwind_start()
409 			(state->next_bp == NULL && state->bp < first_frame)))  in __unwind_start()
410 		unwind_next_frame(state);  in __unwind_start()