14 unsigned long unwind_get_return_address(struct unwind_state *state)  in unwind_get_return_address()  argument
16 	if (unwind_done(state))  in unwind_get_return_address()
19 	return __kernel_text_address(state->ip) ? state->ip : 0;  in unwind_get_return_address()
23 unsigned long *unwind_get_return_address_ptr(struct unwind_state *state)  in unwind_get_return_address_ptr()  argument
25 	if (unwind_done(state))  in unwind_get_return_address_ptr()
28 	return state->regs ? &state->regs->ip : state->bp + 1;  in unwind_get_return_address_ptr()
31 static void unwind_dump(struct unwind_state *state)  in unwind_dump()  argument
45 			state->stack_info.type, state->stack_info.next_sp,  in unwind_dump()
46 			state->stack_mask, state->graph_idx);  in unwind_dump()
48 	for (sp = PTR_ALIGN(state->orig_sp, sizeof(long)); sp;  in unwind_dump()
50 		if (get_stack_info(sp, state->task, &stack_info, &visit_mask))  in unwind_dump()
80 static inline unsigned long *last_frame(struct unwind_state *state)  in last_frame()  argument
82 	return (unsigned long *)task_pt_regs(state->task) - 2;  in last_frame()
85 static bool is_last_frame(struct unwind_state *state)  in is_last_frame()  argument
87 	return state->bp == last_frame(state);  in is_last_frame()
96 static inline unsigned long *last_aligned_frame(struct unwind_state *state)  in last_aligned_frame()  argument
98 	return last_frame(state) - GCC_REALIGN_WORDS;  in last_aligned_frame()
101 static bool is_last_aligned_frame(struct unwind_state *state)  in is_last_aligned_frame()  argument
103 	unsigned long *last_bp = last_frame(state);  in is_last_aligned_frame()
104 	unsigned long *aligned_bp = last_aligned_frame(state);  in is_last_aligned_frame()
129 	return (state->bp == aligned_bp && *(aligned_bp + 1) == *(last_bp + 1));  in is_last_aligned_frame()
132 static bool is_last_ftrace_frame(struct unwind_state *state)  in is_last_ftrace_frame()  argument
134 	unsigned long *last_bp = last_frame(state);  in is_last_ftrace_frame()
149 	return (state->bp == last_ftrace_bp &&  in is_last_ftrace_frame()
150 		*state->bp == *(state->bp + 2) &&  in is_last_ftrace_frame()
151 		*(state->bp + 1) == *(state->bp + 4));  in is_last_ftrace_frame()
154 static bool is_last_task_frame(struct unwind_state *state)  in is_last_task_frame()  argument
156 	return is_last_frame(state) || is_last_aligned_frame(state) ||  in is_last_task_frame()
157 	       is_last_ftrace_frame(state);  in is_last_task_frame()
196 static bool update_stack_state(struct unwind_state *state,  in update_stack_state()  argument
199 	struct stack_info *info = &state->stack_info;  in update_stack_state()
205 	if (state->regs)  in update_stack_state()
206 		prev_frame_end = (void *)state->regs + sizeof(*state->regs);  in update_stack_state()
208 		prev_frame_end = (void *)state->bp + FRAME_HEADER_SIZE;  in update_stack_state()
215 		state->got_irq = true;  in update_stack_state()
229 		if (get_stack_info(info->next_sp, state->task, info,  in update_stack_state()
230 				   &state->stack_mask))  in update_stack_state()
234 	if (state->orig_sp && state->stack_info.type == prev_type &&  in update_stack_state()
240 		state->regs = regs;  in update_stack_state()
241 		state->bp = NULL;  in update_stack_state()
243 		state->bp = next_bp;  in update_stack_state()
244 		state->regs = NULL;  in update_stack_state()
248 	if (state->regs && user_mode(state->regs))  in update_stack_state()
249 		state->ip = 0;  in update_stack_state()
251 		addr_p = unwind_get_return_address_ptr(state);  in update_stack_state()
252 		addr = READ_ONCE_TASK_STACK(state->task, *addr_p);  in update_stack_state()
253 		state->ip = unwind_recover_ret_addr(state, addr, addr_p);  in update_stack_state()
257 	if (!state->orig_sp)  in update_stack_state()
258 		state->orig_sp = frame;  in update_stack_state()
264 bool unwind_next_frame(struct unwind_state *state)  in unwind_next_frame()  argument
269 	if (unwind_done(state))  in unwind_next_frame()
273 	if (state->regs && user_mode(state->regs))  in unwind_next_frame()
276 	if (is_last_task_frame(state)) {  in unwind_next_frame()
277 		regs = task_pt_regs(state->task);  in unwind_next_frame()
298 		state->regs = regs;  in unwind_next_frame()
299 		state->bp = NULL;  in unwind_next_frame()
300 		state->ip = 0;  in unwind_next_frame()
305 	if (state->next_bp) {  in unwind_next_frame()
306 		next_bp = state->next_bp;  in unwind_next_frame()
307 		state->next_bp = NULL;  in unwind_next_frame()
308 	} else if (state->regs) {  in unwind_next_frame()
309 		next_bp = (unsigned long *)state->regs->bp;  in unwind_next_frame()
311 		next_bp = (unsigned long *)READ_ONCE_TASK_STACK(state->task, *state->bp);  in unwind_next_frame()
315 	if (!update_stack_state(state, next_bp))  in unwind_next_frame()
321 	state->error = true;  in unwind_next_frame()
330 	if (state->task != current)  in unwind_next_frame()
337 	if (state->got_irq && in_entry_code(state->ip))  in unwind_next_frame()
339 	if (state->regs &&  in unwind_next_frame()
340 	    state->regs->sp >= (unsigned long)last_aligned_frame(state) &&  in unwind_next_frame()
341 	    state->regs->sp < (unsigned long)task_pt_regs(state->task))  in unwind_next_frame()
351 	if (state->task != current)  in unwind_next_frame()
354 	if (state->regs) {  in unwind_next_frame()
357 			state->regs, state->task->comm,  in unwind_next_frame()
358 			state->task->pid, next_bp);  in unwind_next_frame()
359 		unwind_dump(state);  in unwind_next_frame()
363 			state->bp, state->task->comm,  in unwind_next_frame()
364 			state->task->pid, next_bp);  in unwind_next_frame()
365 		unwind_dump(state);  in unwind_next_frame()
368 	state->stack_info.type = STACK_TYPE_UNKNOWN;  in unwind_next_frame()
373 void __unwind_start(struct unwind_state *state, struct task_struct *task,  in __unwind_start()  argument
378 	memset(state, 0, sizeof(*state));  in __unwind_start()
379 	state->task = task;  in __unwind_start()
380 	state->got_irq = (regs);  in __unwind_start()
384 		state->stack_info.type = STACK_TYPE_UNKNOWN;  in __unwind_start()
400 		state->next_bp = bp;  in __unwind_start()
405 	get_stack_info(bp, state->task, &state->stack_info,  in __unwind_start()
406 		       &state->stack_mask);  in __unwind_start()
407 	update_stack_state(state, bp);  in __unwind_start()
414 	while (!unwind_done(state) &&  in __unwind_start()
415 	       (!on_stack(&state->stack_info, first_frame, sizeof(long)) ||  in __unwind_start()
416 			(state->next_bp == NULL && state->bp < first_frame)))  in __unwind_start()
417 		unwind_next_frame(state);  in __unwind_start()