1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (C) 2015-2017 Josh Poimboeuf <jpoimboe@redhat.com>
4  */
5 
6 #include <string.h>
7 #include <stdlib.h>
8 #include <inttypes.h>
9 #include <sys/mman.h>
10 
11 #include <arch/elf.h>
12 #include <objtool/builtin.h>
13 #include <objtool/cfi.h>
14 #include <objtool/arch.h>
15 #include <objtool/check.h>
16 #include <objtool/special.h>
17 #include <objtool/warn.h>
18 #include <objtool/endianness.h>
19 
20 #include <linux/objtool.h>
21 #include <linux/hashtable.h>
22 #include <linux/kernel.h>
23 #include <linux/static_call_types.h>
24 
25 struct alternative {
26 	struct list_head list;
27 	struct instruction *insn;
28 	bool skip_orig;
29 };
30 
31 static unsigned long nr_cfi, nr_cfi_reused, nr_cfi_cache;
32 
33 static struct cfi_init_state initial_func_cfi;
34 static struct cfi_state init_cfi;
35 static struct cfi_state func_cfi;
36 
find_insn(struct objtool_file * file,struct section * sec,unsigned long offset)37 struct instruction *find_insn(struct objtool_file *file,
38 			      struct section *sec, unsigned long offset)
39 {
40 	struct instruction *insn;
41 
42 	hash_for_each_possible(file->insn_hash, insn, hash, sec_offset_hash(sec, offset)) {
43 		if (insn->sec == sec && insn->offset == offset)
44 			return insn;
45 	}
46 
47 	return NULL;
48 }
49 
next_insn_same_sec(struct objtool_file * file,struct instruction * insn)50 static struct instruction *next_insn_same_sec(struct objtool_file *file,
51 					      struct instruction *insn)
52 {
53 	struct instruction *next = list_next_entry(insn, list);
54 
55 	if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
56 		return NULL;
57 
58 	return next;
59 }
60 
next_insn_same_func(struct objtool_file * file,struct instruction * insn)61 static struct instruction *next_insn_same_func(struct objtool_file *file,
62 					       struct instruction *insn)
63 {
64 	struct instruction *next = list_next_entry(insn, list);
65 	struct symbol *func = insn->func;
66 
67 	if (!func)
68 		return NULL;
69 
70 	if (&next->list != &file->insn_list && next->func == func)
71 		return next;
72 
73 	/* Check if we're already in the subfunction: */
74 	if (func == func->cfunc)
75 		return NULL;
76 
77 	/* Move to the subfunction: */
78 	return find_insn(file, func->cfunc->sec, func->cfunc->offset);
79 }
80 
prev_insn_same_sym(struct objtool_file * file,struct instruction * insn)81 static struct instruction *prev_insn_same_sym(struct objtool_file *file,
82 					       struct instruction *insn)
83 {
84 	struct instruction *prev = list_prev_entry(insn, list);
85 
86 	if (&prev->list != &file->insn_list && prev->func == insn->func)
87 		return prev;
88 
89 	return NULL;
90 }
91 
92 #define func_for_each_insn(file, func, insn)				\
93 	for (insn = find_insn(file, func->sec, func->offset);		\
94 	     insn;							\
95 	     insn = next_insn_same_func(file, insn))
96 
97 #define sym_for_each_insn(file, sym, insn)				\
98 	for (insn = find_insn(file, sym->sec, sym->offset);		\
99 	     insn && &insn->list != &file->insn_list &&			\
100 		insn->sec == sym->sec &&				\
101 		insn->offset < sym->offset + sym->len;			\
102 	     insn = list_next_entry(insn, list))
103 
104 #define sym_for_each_insn_continue_reverse(file, sym, insn)		\
105 	for (insn = list_prev_entry(insn, list);			\
106 	     &insn->list != &file->insn_list &&				\
107 		insn->sec == sym->sec && insn->offset >= sym->offset;	\
108 	     insn = list_prev_entry(insn, list))
109 
110 #define sec_for_each_insn_from(file, insn)				\
111 	for (; insn; insn = next_insn_same_sec(file, insn))
112 
113 #define sec_for_each_insn_continue(file, insn)				\
114 	for (insn = next_insn_same_sec(file, insn); insn;		\
115 	     insn = next_insn_same_sec(file, insn))
116 
is_jump_table_jump(struct instruction * insn)117 static bool is_jump_table_jump(struct instruction *insn)
118 {
119 	struct alt_group *alt_group = insn->alt_group;
120 
121 	if (insn->jump_table)
122 		return true;
123 
124 	/* Retpoline alternative for a jump table? */
125 	return alt_group && alt_group->orig_group &&
126 	       alt_group->orig_group->first_insn->jump_table;
127 }
128 
is_sibling_call(struct instruction * insn)129 static bool is_sibling_call(struct instruction *insn)
130 {
131 	/*
132 	 * Assume only ELF functions can make sibling calls.  This ensures
133 	 * sibling call detection consistency between vmlinux.o and individual
134 	 * objects.
135 	 */
136 	if (!insn->func)
137 		return false;
138 
139 	/* An indirect jump is either a sibling call or a jump to a table. */
140 	if (insn->type == INSN_JUMP_DYNAMIC)
141 		return !is_jump_table_jump(insn);
142 
143 	/* add_jump_destinations() sets insn->call_dest for sibling calls. */
144 	return (is_static_jump(insn) && insn->call_dest);
145 }
146 
147 /*
148  * This checks to see if the given function is a "noreturn" function.
149  *
150  * For global functions which are outside the scope of this object file, we
151  * have to keep a manual list of them.
152  *
153  * For local functions, we have to detect them manually by simply looking for
154  * the lack of a return instruction.
155  */
__dead_end_function(struct objtool_file * file,struct symbol * func,int recursion)156 static bool __dead_end_function(struct objtool_file *file, struct symbol *func,
157 				int recursion)
158 {
159 	int i;
160 	struct instruction *insn;
161 	bool empty = true;
162 
163 	/*
164 	 * Unfortunately these have to be hard coded because the noreturn
165 	 * attribute isn't provided in ELF data. Keep 'em sorted.
166 	 */
167 	static const char * const global_noreturns[] = {
168 		"__invalid_creds",
169 		"__module_put_and_kthread_exit",
170 		"__reiserfs_panic",
171 		"__stack_chk_fail",
172 		"__ubsan_handle_builtin_unreachable",
173 		"cpu_bringup_and_idle",
174 		"cpu_startup_entry",
175 		"do_exit",
176 		"do_group_exit",
177 		"do_task_dead",
178 		"ex_handler_msr_mce",
179 		"fortify_panic",
180 		"kthread_complete_and_exit",
181 		"kthread_exit",
182 		"kunit_try_catch_throw",
183 		"lbug_with_loc",
184 		"machine_real_restart",
185 		"make_task_dead",
186 		"panic",
187 		"rewind_stack_and_make_dead",
188 		"sev_es_terminate",
189 		"snp_abort",
190 		"stop_this_cpu",
191 		"usercopy_abort",
192 		"xen_start_kernel",
193 	};
194 
195 	if (!func)
196 		return false;
197 
198 	if (func->bind == STB_WEAK)
199 		return false;
200 
201 	if (func->bind == STB_GLOBAL)
202 		for (i = 0; i < ARRAY_SIZE(global_noreturns); i++)
203 			if (!strcmp(func->name, global_noreturns[i]))
204 				return true;
205 
206 	if (!func->len)
207 		return false;
208 
209 	insn = find_insn(file, func->sec, func->offset);
210 	if (!insn || !insn->func)
211 		return false;
212 
213 	func_for_each_insn(file, func, insn) {
214 		empty = false;
215 
216 		if (insn->type == INSN_RETURN)
217 			return false;
218 	}
219 
220 	if (empty)
221 		return false;
222 
223 	/*
224 	 * A function can have a sibling call instead of a return.  In that
225 	 * case, the function's dead-end status depends on whether the target
226 	 * of the sibling call returns.
227 	 */
228 	func_for_each_insn(file, func, insn) {
229 		if (is_sibling_call(insn)) {
230 			struct instruction *dest = insn->jump_dest;
231 
232 			if (!dest)
233 				/* sibling call to another file */
234 				return false;
235 
236 			/* local sibling call */
237 			if (recursion == 5) {
238 				/*
239 				 * Infinite recursion: two functions have
240 				 * sibling calls to each other.  This is a very
241 				 * rare case.  It means they aren't dead ends.
242 				 */
243 				return false;
244 			}
245 
246 			return __dead_end_function(file, dest->func, recursion+1);
247 		}
248 	}
249 
250 	return true;
251 }
252 
dead_end_function(struct objtool_file * file,struct symbol * func)253 static bool dead_end_function(struct objtool_file *file, struct symbol *func)
254 {
255 	return __dead_end_function(file, func, 0);
256 }
257 
init_cfi_state(struct cfi_state * cfi)258 static void init_cfi_state(struct cfi_state *cfi)
259 {
260 	int i;
261 
262 	for (i = 0; i < CFI_NUM_REGS; i++) {
263 		cfi->regs[i].base = CFI_UNDEFINED;
264 		cfi->vals[i].base = CFI_UNDEFINED;
265 	}
266 	cfi->cfa.base = CFI_UNDEFINED;
267 	cfi->drap_reg = CFI_UNDEFINED;
268 	cfi->drap_offset = -1;
269 }
270 
init_insn_state(struct objtool_file * file,struct insn_state * state,struct section * sec)271 static void init_insn_state(struct objtool_file *file, struct insn_state *state,
272 			    struct section *sec)
273 {
274 	memset(state, 0, sizeof(*state));
275 	init_cfi_state(&state->cfi);
276 
277 	/*
278 	 * We need the full vmlinux for noinstr validation, otherwise we can
279 	 * not correctly determine insn->call_dest->sec (external symbols do
280 	 * not have a section).
281 	 */
282 	if (opts.link && opts.noinstr && sec)
283 		state->noinstr = sec->noinstr;
284 }
285 
cfi_alloc(void)286 static struct cfi_state *cfi_alloc(void)
287 {
288 	struct cfi_state *cfi = calloc(sizeof(struct cfi_state), 1);
289 	if (!cfi) {
290 		WARN("calloc failed");
291 		exit(1);
292 	}
293 	nr_cfi++;
294 	return cfi;
295 }
296 
297 static int cfi_bits;
298 static struct hlist_head *cfi_hash;
299 
cficmp(struct cfi_state * cfi1,struct cfi_state * cfi2)300 static inline bool cficmp(struct cfi_state *cfi1, struct cfi_state *cfi2)
301 {
302 	return memcmp((void *)cfi1 + sizeof(cfi1->hash),
303 		      (void *)cfi2 + sizeof(cfi2->hash),
304 		      sizeof(struct cfi_state) - sizeof(struct hlist_node));
305 }
306 
cfi_key(struct cfi_state * cfi)307 static inline u32 cfi_key(struct cfi_state *cfi)
308 {
309 	return jhash((void *)cfi + sizeof(cfi->hash),
310 		     sizeof(*cfi) - sizeof(cfi->hash), 0);
311 }
312 
cfi_hash_find_or_add(struct cfi_state * cfi)313 static struct cfi_state *cfi_hash_find_or_add(struct cfi_state *cfi)
314 {
315 	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
316 	struct cfi_state *obj;
317 
318 	hlist_for_each_entry(obj, head, hash) {
319 		if (!cficmp(cfi, obj)) {
320 			nr_cfi_cache++;
321 			return obj;
322 		}
323 	}
324 
325 	obj = cfi_alloc();
326 	*obj = *cfi;
327 	hlist_add_head(&obj->hash, head);
328 
329 	return obj;
330 }
331 
cfi_hash_add(struct cfi_state * cfi)332 static void cfi_hash_add(struct cfi_state *cfi)
333 {
334 	struct hlist_head *head = &cfi_hash[hash_min(cfi_key(cfi), cfi_bits)];
335 
336 	hlist_add_head(&cfi->hash, head);
337 }
338 
cfi_hash_alloc(unsigned long size)339 static void *cfi_hash_alloc(unsigned long size)
340 {
341 	cfi_bits = max(10, ilog2(size));
342 	cfi_hash = mmap(NULL, sizeof(struct hlist_head) << cfi_bits,
343 			PROT_READ|PROT_WRITE,
344 			MAP_PRIVATE|MAP_ANON, -1, 0);
345 	if (cfi_hash == (void *)-1L) {
346 		WARN("mmap fail cfi_hash");
347 		cfi_hash = NULL;
348 	}  else if (opts.stats) {
349 		printf("cfi_bits: %d\n", cfi_bits);
350 	}
351 
352 	return cfi_hash;
353 }
354 
355 static unsigned long nr_insns;
356 static unsigned long nr_insns_visited;
357 
358 /*
359  * Call the arch-specific instruction decoder for all the instructions and add
360  * them to the global instruction list.
361  */
decode_instructions(struct objtool_file * file)362 static int decode_instructions(struct objtool_file *file)
363 {
364 	struct section *sec;
365 	struct symbol *func;
366 	unsigned long offset;
367 	struct instruction *insn;
368 	int ret;
369 
370 	for_each_sec(file, sec) {
371 
372 		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
373 			continue;
374 
375 		if (strcmp(sec->name, ".altinstr_replacement") &&
376 		    strcmp(sec->name, ".altinstr_aux") &&
377 		    strncmp(sec->name, ".discard.", 9))
378 			sec->text = true;
379 
380 		if (!strcmp(sec->name, ".noinstr.text") ||
381 		    !strcmp(sec->name, ".entry.text") ||
382 		    !strncmp(sec->name, ".text.__x86.", 12))
383 			sec->noinstr = true;
384 
385 		for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
386 			insn = malloc(sizeof(*insn));
387 			if (!insn) {
388 				WARN("malloc failed");
389 				return -1;
390 			}
391 			memset(insn, 0, sizeof(*insn));
392 			INIT_LIST_HEAD(&insn->alts);
393 			INIT_LIST_HEAD(&insn->stack_ops);
394 			INIT_LIST_HEAD(&insn->call_node);
395 
396 			insn->sec = sec;
397 			insn->offset = offset;
398 
399 			ret = arch_decode_instruction(file, sec, offset,
400 						      sec->sh.sh_size - offset,
401 						      &insn->len, &insn->type,
402 						      &insn->immediate,
403 						      &insn->stack_ops);
404 			if (ret)
405 				goto err;
406 
407 			/*
408 			 * By default, "ud2" is a dead end unless otherwise
409 			 * annotated, because GCC 7 inserts it for certain
410 			 * divide-by-zero cases.
411 			 */
412 			if (insn->type == INSN_BUG)
413 				insn->dead_end = true;
414 
415 			hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
416 			list_add_tail(&insn->list, &file->insn_list);
417 			nr_insns++;
418 		}
419 
420 		list_for_each_entry(func, &sec->symbol_list, list) {
421 			if (func->type != STT_FUNC || func->alias != func)
422 				continue;
423 
424 			if (!find_insn(file, sec, func->offset)) {
425 				WARN("%s(): can't find starting instruction",
426 				     func->name);
427 				return -1;
428 			}
429 
430 			sym_for_each_insn(file, func, insn) {
431 				insn->func = func;
432 				if (insn->type == INSN_ENDBR && list_empty(&insn->call_node)) {
433 					if (insn->offset == insn->func->offset) {
434 						list_add_tail(&insn->call_node, &file->endbr_list);
435 						file->nr_endbr++;
436 					} else {
437 						file->nr_endbr_int++;
438 					}
439 				}
440 			}
441 		}
442 	}
443 
444 	if (opts.stats)
445 		printf("nr_insns: %lu\n", nr_insns);
446 
447 	return 0;
448 
449 err:
450 	free(insn);
451 	return ret;
452 }
453 
454 /*
455  * Read the pv_ops[] .data table to find the static initialized values.
456  */
add_pv_ops(struct objtool_file * file,const char * symname)457 static int add_pv_ops(struct objtool_file *file, const char *symname)
458 {
459 	struct symbol *sym, *func;
460 	unsigned long off, end;
461 	struct reloc *rel;
462 	int idx;
463 
464 	sym = find_symbol_by_name(file->elf, symname);
465 	if (!sym)
466 		return 0;
467 
468 	off = sym->offset;
469 	end = off + sym->len;
470 	for (;;) {
471 		rel = find_reloc_by_dest_range(file->elf, sym->sec, off, end - off);
472 		if (!rel)
473 			break;
474 
475 		func = rel->sym;
476 		if (func->type == STT_SECTION)
477 			func = find_symbol_by_offset(rel->sym->sec, rel->addend);
478 
479 		idx = (rel->offset - sym->offset) / sizeof(unsigned long);
480 
481 		objtool_pv_add(file, idx, func);
482 
483 		off = rel->offset + 1;
484 		if (off > end)
485 			break;
486 	}
487 
488 	return 0;
489 }
490 
491 /*
492  * Allocate and initialize file->pv_ops[].
493  */
init_pv_ops(struct objtool_file * file)494 static int init_pv_ops(struct objtool_file *file)
495 {
496 	static const char *pv_ops_tables[] = {
497 		"pv_ops",
498 		"xen_cpu_ops",
499 		"xen_irq_ops",
500 		"xen_mmu_ops",
501 		NULL,
502 	};
503 	const char *pv_ops;
504 	struct symbol *sym;
505 	int idx, nr;
506 
507 	if (!opts.noinstr)
508 		return 0;
509 
510 	file->pv_ops = NULL;
511 
512 	sym = find_symbol_by_name(file->elf, "pv_ops");
513 	if (!sym)
514 		return 0;
515 
516 	nr = sym->len / sizeof(unsigned long);
517 	file->pv_ops = calloc(sizeof(struct pv_state), nr);
518 	if (!file->pv_ops)
519 		return -1;
520 
521 	for (idx = 0; idx < nr; idx++)
522 		INIT_LIST_HEAD(&file->pv_ops[idx].targets);
523 
524 	for (idx = 0; (pv_ops = pv_ops_tables[idx]); idx++)
525 		add_pv_ops(file, pv_ops);
526 
527 	return 0;
528 }
529 
find_last_insn(struct objtool_file * file,struct section * sec)530 static struct instruction *find_last_insn(struct objtool_file *file,
531 					  struct section *sec)
532 {
533 	struct instruction *insn = NULL;
534 	unsigned int offset;
535 	unsigned int end = (sec->sh.sh_size > 10) ? sec->sh.sh_size - 10 : 0;
536 
537 	for (offset = sec->sh.sh_size - 1; offset >= end && !insn; offset--)
538 		insn = find_insn(file, sec, offset);
539 
540 	return insn;
541 }
542 
543 /*
544  * Mark "ud2" instructions and manually annotated dead ends.
545  */
add_dead_ends(struct objtool_file * file)546 static int add_dead_ends(struct objtool_file *file)
547 {
548 	struct section *sec;
549 	struct reloc *reloc;
550 	struct instruction *insn;
551 
552 	/*
553 	 * Check for manually annotated dead ends.
554 	 */
555 	sec = find_section_by_name(file->elf, ".rela.discard.unreachable");
556 	if (!sec)
557 		goto reachable;
558 
559 	list_for_each_entry(reloc, &sec->reloc_list, list) {
560 		if (reloc->sym->type != STT_SECTION) {
561 			WARN("unexpected relocation symbol type in %s", sec->name);
562 			return -1;
563 		}
564 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
565 		if (insn)
566 			insn = list_prev_entry(insn, list);
567 		else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
568 			insn = find_last_insn(file, reloc->sym->sec);
569 			if (!insn) {
570 				WARN("can't find unreachable insn at %s+0x%" PRIx64,
571 				     reloc->sym->sec->name, reloc->addend);
572 				return -1;
573 			}
574 		} else {
575 			WARN("can't find unreachable insn at %s+0x%" PRIx64,
576 			     reloc->sym->sec->name, reloc->addend);
577 			return -1;
578 		}
579 
580 		insn->dead_end = true;
581 	}
582 
583 reachable:
584 	/*
585 	 * These manually annotated reachable checks are needed for GCC 4.4,
586 	 * where the Linux unreachable() macro isn't supported.  In that case
587 	 * GCC doesn't know the "ud2" is fatal, so it generates code as if it's
588 	 * not a dead end.
589 	 */
590 	sec = find_section_by_name(file->elf, ".rela.discard.reachable");
591 	if (!sec)
592 		return 0;
593 
594 	list_for_each_entry(reloc, &sec->reloc_list, list) {
595 		if (reloc->sym->type != STT_SECTION) {
596 			WARN("unexpected relocation symbol type in %s", sec->name);
597 			return -1;
598 		}
599 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
600 		if (insn)
601 			insn = list_prev_entry(insn, list);
602 		else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
603 			insn = find_last_insn(file, reloc->sym->sec);
604 			if (!insn) {
605 				WARN("can't find reachable insn at %s+0x%" PRIx64,
606 				     reloc->sym->sec->name, reloc->addend);
607 				return -1;
608 			}
609 		} else {
610 			WARN("can't find reachable insn at %s+0x%" PRIx64,
611 			     reloc->sym->sec->name, reloc->addend);
612 			return -1;
613 		}
614 
615 		insn->dead_end = false;
616 	}
617 
618 	return 0;
619 }
620 
create_static_call_sections(struct objtool_file * file)621 static int create_static_call_sections(struct objtool_file *file)
622 {
623 	struct section *sec;
624 	struct static_call_site *site;
625 	struct instruction *insn;
626 	struct symbol *key_sym;
627 	char *key_name, *tmp;
628 	int idx;
629 
630 	sec = find_section_by_name(file->elf, ".static_call_sites");
631 	if (sec) {
632 		INIT_LIST_HEAD(&file->static_call_list);
633 		WARN("file already has .static_call_sites section, skipping");
634 		return 0;
635 	}
636 
637 	if (list_empty(&file->static_call_list))
638 		return 0;
639 
640 	idx = 0;
641 	list_for_each_entry(insn, &file->static_call_list, call_node)
642 		idx++;
643 
644 	sec = elf_create_section(file->elf, ".static_call_sites", SHF_WRITE,
645 				 sizeof(struct static_call_site), idx);
646 	if (!sec)
647 		return -1;
648 
649 	idx = 0;
650 	list_for_each_entry(insn, &file->static_call_list, call_node) {
651 
652 		site = (struct static_call_site *)sec->data->d_buf + idx;
653 		memset(site, 0, sizeof(struct static_call_site));
654 
655 		/* populate reloc for 'addr' */
656 		if (elf_add_reloc_to_insn(file->elf, sec,
657 					  idx * sizeof(struct static_call_site),
658 					  R_X86_64_PC32,
659 					  insn->sec, insn->offset))
660 			return -1;
661 
662 		/* find key symbol */
663 		key_name = strdup(insn->call_dest->name);
664 		if (!key_name) {
665 			perror("strdup");
666 			return -1;
667 		}
668 		if (strncmp(key_name, STATIC_CALL_TRAMP_PREFIX_STR,
669 			    STATIC_CALL_TRAMP_PREFIX_LEN)) {
670 			WARN("static_call: trampoline name malformed: %s", key_name);
671 			return -1;
672 		}
673 		tmp = key_name + STATIC_CALL_TRAMP_PREFIX_LEN - STATIC_CALL_KEY_PREFIX_LEN;
674 		memcpy(tmp, STATIC_CALL_KEY_PREFIX_STR, STATIC_CALL_KEY_PREFIX_LEN);
675 
676 		key_sym = find_symbol_by_name(file->elf, tmp);
677 		if (!key_sym) {
678 			if (!opts.module) {
679 				WARN("static_call: can't find static_call_key symbol: %s", tmp);
680 				return -1;
681 			}
682 
683 			/*
684 			 * For modules(), the key might not be exported, which
685 			 * means the module can make static calls but isn't
686 			 * allowed to change them.
687 			 *
688 			 * In that case we temporarily set the key to be the
689 			 * trampoline address.  This is fixed up in
690 			 * static_call_add_module().
691 			 */
692 			key_sym = insn->call_dest;
693 		}
694 		free(key_name);
695 
696 		/* populate reloc for 'key' */
697 		if (elf_add_reloc(file->elf, sec,
698 				  idx * sizeof(struct static_call_site) + 4,
699 				  R_X86_64_PC32, key_sym,
700 				  is_sibling_call(insn) * STATIC_CALL_SITE_TAIL))
701 			return -1;
702 
703 		idx++;
704 	}
705 
706 	return 0;
707 }
708 
create_retpoline_sites_sections(struct objtool_file * file)709 static int create_retpoline_sites_sections(struct objtool_file *file)
710 {
711 	struct instruction *insn;
712 	struct section *sec;
713 	int idx;
714 
715 	sec = find_section_by_name(file->elf, ".retpoline_sites");
716 	if (sec) {
717 		WARN("file already has .retpoline_sites, skipping");
718 		return 0;
719 	}
720 
721 	idx = 0;
722 	list_for_each_entry(insn, &file->retpoline_call_list, call_node)
723 		idx++;
724 
725 	if (!idx)
726 		return 0;
727 
728 	sec = elf_create_section(file->elf, ".retpoline_sites", 0,
729 				 sizeof(int), idx);
730 	if (!sec) {
731 		WARN("elf_create_section: .retpoline_sites");
732 		return -1;
733 	}
734 
735 	idx = 0;
736 	list_for_each_entry(insn, &file->retpoline_call_list, call_node) {
737 
738 		int *site = (int *)sec->data->d_buf + idx;
739 		*site = 0;
740 
741 		if (elf_add_reloc_to_insn(file->elf, sec,
742 					  idx * sizeof(int),
743 					  R_X86_64_PC32,
744 					  insn->sec, insn->offset)) {
745 			WARN("elf_add_reloc_to_insn: .retpoline_sites");
746 			return -1;
747 		}
748 
749 		idx++;
750 	}
751 
752 	return 0;
753 }
754 
create_return_sites_sections(struct objtool_file * file)755 static int create_return_sites_sections(struct objtool_file *file)
756 {
757 	struct instruction *insn;
758 	struct section *sec;
759 	int idx;
760 
761 	sec = find_section_by_name(file->elf, ".return_sites");
762 	if (sec) {
763 		WARN("file already has .return_sites, skipping");
764 		return 0;
765 	}
766 
767 	idx = 0;
768 	list_for_each_entry(insn, &file->return_thunk_list, call_node)
769 		idx++;
770 
771 	if (!idx)
772 		return 0;
773 
774 	sec = elf_create_section(file->elf, ".return_sites", 0,
775 				 sizeof(int), idx);
776 	if (!sec) {
777 		WARN("elf_create_section: .return_sites");
778 		return -1;
779 	}
780 
781 	idx = 0;
782 	list_for_each_entry(insn, &file->return_thunk_list, call_node) {
783 
784 		int *site = (int *)sec->data->d_buf + idx;
785 		*site = 0;
786 
787 		if (elf_add_reloc_to_insn(file->elf, sec,
788 					  idx * sizeof(int),
789 					  R_X86_64_PC32,
790 					  insn->sec, insn->offset)) {
791 			WARN("elf_add_reloc_to_insn: .return_sites");
792 			return -1;
793 		}
794 
795 		idx++;
796 	}
797 
798 	return 0;
799 }
800 
create_ibt_endbr_seal_sections(struct objtool_file * file)801 static int create_ibt_endbr_seal_sections(struct objtool_file *file)
802 {
803 	struct instruction *insn;
804 	struct section *sec;
805 	int idx;
806 
807 	sec = find_section_by_name(file->elf, ".ibt_endbr_seal");
808 	if (sec) {
809 		WARN("file already has .ibt_endbr_seal, skipping");
810 		return 0;
811 	}
812 
813 	idx = 0;
814 	list_for_each_entry(insn, &file->endbr_list, call_node)
815 		idx++;
816 
817 	if (opts.stats) {
818 		printf("ibt: ENDBR at function start: %d\n", file->nr_endbr);
819 		printf("ibt: ENDBR inside functions:  %d\n", file->nr_endbr_int);
820 		printf("ibt: superfluous ENDBR:       %d\n", idx);
821 	}
822 
823 	if (!idx)
824 		return 0;
825 
826 	sec = elf_create_section(file->elf, ".ibt_endbr_seal", 0,
827 				 sizeof(int), idx);
828 	if (!sec) {
829 		WARN("elf_create_section: .ibt_endbr_seal");
830 		return -1;
831 	}
832 
833 	idx = 0;
834 	list_for_each_entry(insn, &file->endbr_list, call_node) {
835 
836 		int *site = (int *)sec->data->d_buf + idx;
837 		*site = 0;
838 
839 		if (elf_add_reloc_to_insn(file->elf, sec,
840 					  idx * sizeof(int),
841 					  R_X86_64_PC32,
842 					  insn->sec, insn->offset)) {
843 			WARN("elf_add_reloc_to_insn: .ibt_endbr_seal");
844 			return -1;
845 		}
846 
847 		idx++;
848 	}
849 
850 	return 0;
851 }
852 
create_mcount_loc_sections(struct objtool_file * file)853 static int create_mcount_loc_sections(struct objtool_file *file)
854 {
855 	struct section *sec;
856 	unsigned long *loc;
857 	struct instruction *insn;
858 	int idx;
859 
860 	sec = find_section_by_name(file->elf, "__mcount_loc");
861 	if (sec) {
862 		INIT_LIST_HEAD(&file->mcount_loc_list);
863 		WARN("file already has __mcount_loc section, skipping");
864 		return 0;
865 	}
866 
867 	if (list_empty(&file->mcount_loc_list))
868 		return 0;
869 
870 	idx = 0;
871 	list_for_each_entry(insn, &file->mcount_loc_list, call_node)
872 		idx++;
873 
874 	sec = elf_create_section(file->elf, "__mcount_loc", 0, sizeof(unsigned long), idx);
875 	if (!sec)
876 		return -1;
877 
878 	idx = 0;
879 	list_for_each_entry(insn, &file->mcount_loc_list, call_node) {
880 
881 		loc = (unsigned long *)sec->data->d_buf + idx;
882 		memset(loc, 0, sizeof(unsigned long));
883 
884 		if (elf_add_reloc_to_insn(file->elf, sec,
885 					  idx * sizeof(unsigned long),
886 					  R_X86_64_64,
887 					  insn->sec, insn->offset))
888 			return -1;
889 
890 		idx++;
891 	}
892 
893 	return 0;
894 }
895 
896 /*
897  * Warnings shouldn't be reported for ignored functions.
898  */
add_ignores(struct objtool_file * file)899 static void add_ignores(struct objtool_file *file)
900 {
901 	struct instruction *insn;
902 	struct section *sec;
903 	struct symbol *func;
904 	struct reloc *reloc;
905 
906 	sec = find_section_by_name(file->elf, ".rela.discard.func_stack_frame_non_standard");
907 	if (!sec)
908 		return;
909 
910 	list_for_each_entry(reloc, &sec->reloc_list, list) {
911 		switch (reloc->sym->type) {
912 		case STT_FUNC:
913 			func = reloc->sym;
914 			break;
915 
916 		case STT_SECTION:
917 			func = find_func_by_offset(reloc->sym->sec, reloc->addend);
918 			if (!func)
919 				continue;
920 			break;
921 
922 		default:
923 			WARN("unexpected relocation symbol type in %s: %d", sec->name, reloc->sym->type);
924 			continue;
925 		}
926 
927 		func_for_each_insn(file, func, insn)
928 			insn->ignore = true;
929 	}
930 }
931 
932 /*
933  * This is a whitelist of functions that is allowed to be called with AC set.
934  * The list is meant to be minimal and only contains compiler instrumentation
935  * ABI and a few functions used to implement *_{to,from}_user() functions.
936  *
937  * These functions must not directly change AC, but may PUSHF/POPF.
938  */
939 static const char *uaccess_safe_builtin[] = {
940 	/* KASAN */
941 	"kasan_report",
942 	"kasan_check_range",
943 	/* KASAN out-of-line */
944 	"__asan_loadN_noabort",
945 	"__asan_load1_noabort",
946 	"__asan_load2_noabort",
947 	"__asan_load4_noabort",
948 	"__asan_load8_noabort",
949 	"__asan_load16_noabort",
950 	"__asan_storeN_noabort",
951 	"__asan_store1_noabort",
952 	"__asan_store2_noabort",
953 	"__asan_store4_noabort",
954 	"__asan_store8_noabort",
955 	"__asan_store16_noabort",
956 	"__kasan_check_read",
957 	"__kasan_check_write",
958 	/* KASAN in-line */
959 	"__asan_report_load_n_noabort",
960 	"__asan_report_load1_noabort",
961 	"__asan_report_load2_noabort",
962 	"__asan_report_load4_noabort",
963 	"__asan_report_load8_noabort",
964 	"__asan_report_load16_noabort",
965 	"__asan_report_store_n_noabort",
966 	"__asan_report_store1_noabort",
967 	"__asan_report_store2_noabort",
968 	"__asan_report_store4_noabort",
969 	"__asan_report_store8_noabort",
970 	"__asan_report_store16_noabort",
971 	/* KCSAN */
972 	"__kcsan_check_access",
973 	"__kcsan_mb",
974 	"__kcsan_wmb",
975 	"__kcsan_rmb",
976 	"__kcsan_release",
977 	"kcsan_found_watchpoint",
978 	"kcsan_setup_watchpoint",
979 	"kcsan_check_scoped_accesses",
980 	"kcsan_disable_current",
981 	"kcsan_enable_current_nowarn",
982 	/* KCSAN/TSAN */
983 	"__tsan_func_entry",
984 	"__tsan_func_exit",
985 	"__tsan_read_range",
986 	"__tsan_write_range",
987 	"__tsan_read1",
988 	"__tsan_read2",
989 	"__tsan_read4",
990 	"__tsan_read8",
991 	"__tsan_read16",
992 	"__tsan_write1",
993 	"__tsan_write2",
994 	"__tsan_write4",
995 	"__tsan_write8",
996 	"__tsan_write16",
997 	"__tsan_read_write1",
998 	"__tsan_read_write2",
999 	"__tsan_read_write4",
1000 	"__tsan_read_write8",
1001 	"__tsan_read_write16",
1002 	"__tsan_volatile_read1",
1003 	"__tsan_volatile_read2",
1004 	"__tsan_volatile_read4",
1005 	"__tsan_volatile_read8",
1006 	"__tsan_volatile_read16",
1007 	"__tsan_volatile_write1",
1008 	"__tsan_volatile_write2",
1009 	"__tsan_volatile_write4",
1010 	"__tsan_volatile_write8",
1011 	"__tsan_volatile_write16",
1012 	"__tsan_atomic8_load",
1013 	"__tsan_atomic16_load",
1014 	"__tsan_atomic32_load",
1015 	"__tsan_atomic64_load",
1016 	"__tsan_atomic8_store",
1017 	"__tsan_atomic16_store",
1018 	"__tsan_atomic32_store",
1019 	"__tsan_atomic64_store",
1020 	"__tsan_atomic8_exchange",
1021 	"__tsan_atomic16_exchange",
1022 	"__tsan_atomic32_exchange",
1023 	"__tsan_atomic64_exchange",
1024 	"__tsan_atomic8_fetch_add",
1025 	"__tsan_atomic16_fetch_add",
1026 	"__tsan_atomic32_fetch_add",
1027 	"__tsan_atomic64_fetch_add",
1028 	"__tsan_atomic8_fetch_sub",
1029 	"__tsan_atomic16_fetch_sub",
1030 	"__tsan_atomic32_fetch_sub",
1031 	"__tsan_atomic64_fetch_sub",
1032 	"__tsan_atomic8_fetch_and",
1033 	"__tsan_atomic16_fetch_and",
1034 	"__tsan_atomic32_fetch_and",
1035 	"__tsan_atomic64_fetch_and",
1036 	"__tsan_atomic8_fetch_or",
1037 	"__tsan_atomic16_fetch_or",
1038 	"__tsan_atomic32_fetch_or",
1039 	"__tsan_atomic64_fetch_or",
1040 	"__tsan_atomic8_fetch_xor",
1041 	"__tsan_atomic16_fetch_xor",
1042 	"__tsan_atomic32_fetch_xor",
1043 	"__tsan_atomic64_fetch_xor",
1044 	"__tsan_atomic8_fetch_nand",
1045 	"__tsan_atomic16_fetch_nand",
1046 	"__tsan_atomic32_fetch_nand",
1047 	"__tsan_atomic64_fetch_nand",
1048 	"__tsan_atomic8_compare_exchange_strong",
1049 	"__tsan_atomic16_compare_exchange_strong",
1050 	"__tsan_atomic32_compare_exchange_strong",
1051 	"__tsan_atomic64_compare_exchange_strong",
1052 	"__tsan_atomic8_compare_exchange_weak",
1053 	"__tsan_atomic16_compare_exchange_weak",
1054 	"__tsan_atomic32_compare_exchange_weak",
1055 	"__tsan_atomic64_compare_exchange_weak",
1056 	"__tsan_atomic8_compare_exchange_val",
1057 	"__tsan_atomic16_compare_exchange_val",
1058 	"__tsan_atomic32_compare_exchange_val",
1059 	"__tsan_atomic64_compare_exchange_val",
1060 	"__tsan_atomic_thread_fence",
1061 	"__tsan_atomic_signal_fence",
1062 	/* KCOV */
1063 	"write_comp_data",
1064 	"check_kcov_mode",
1065 	"__sanitizer_cov_trace_pc",
1066 	"__sanitizer_cov_trace_const_cmp1",
1067 	"__sanitizer_cov_trace_const_cmp2",
1068 	"__sanitizer_cov_trace_const_cmp4",
1069 	"__sanitizer_cov_trace_const_cmp8",
1070 	"__sanitizer_cov_trace_cmp1",
1071 	"__sanitizer_cov_trace_cmp2",
1072 	"__sanitizer_cov_trace_cmp4",
1073 	"__sanitizer_cov_trace_cmp8",
1074 	"__sanitizer_cov_trace_switch",
1075 	/* KMSAN */
1076 	"kmsan_copy_to_user",
1077 	"kmsan_report",
1078 	"kmsan_unpoison_entry_regs",
1079 	"kmsan_unpoison_memory",
1080 	"__msan_chain_origin",
1081 	"__msan_get_context_state",
1082 	"__msan_instrument_asm_store",
1083 	"__msan_metadata_ptr_for_load_1",
1084 	"__msan_metadata_ptr_for_load_2",
1085 	"__msan_metadata_ptr_for_load_4",
1086 	"__msan_metadata_ptr_for_load_8",
1087 	"__msan_metadata_ptr_for_load_n",
1088 	"__msan_metadata_ptr_for_store_1",
1089 	"__msan_metadata_ptr_for_store_2",
1090 	"__msan_metadata_ptr_for_store_4",
1091 	"__msan_metadata_ptr_for_store_8",
1092 	"__msan_metadata_ptr_for_store_n",
1093 	"__msan_poison_alloca",
1094 	"__msan_warning",
1095 	/* UBSAN */
1096 	"ubsan_type_mismatch_common",
1097 	"__ubsan_handle_type_mismatch",
1098 	"__ubsan_handle_type_mismatch_v1",
1099 	"__ubsan_handle_shift_out_of_bounds",
1100 	/* misc */
1101 	"csum_partial_copy_generic",
1102 	"copy_mc_fragile",
1103 	"copy_mc_fragile_handle_tail",
1104 	"copy_mc_enhanced_fast_string",
1105 	"ftrace_likely_update", /* CONFIG_TRACE_BRANCH_PROFILING */
1106 	"clear_user_erms",
1107 	"clear_user_rep_good",
1108 	"clear_user_original",
1109 	NULL
1110 };
1111 
add_uaccess_safe(struct objtool_file * file)1112 static void add_uaccess_safe(struct objtool_file *file)
1113 {
1114 	struct symbol *func;
1115 	const char **name;
1116 
1117 	if (!opts.uaccess)
1118 		return;
1119 
1120 	for (name = uaccess_safe_builtin; *name; name++) {
1121 		func = find_symbol_by_name(file->elf, *name);
1122 		if (!func)
1123 			continue;
1124 
1125 		func->uaccess_safe = true;
1126 	}
1127 }
1128 
1129 /*
1130  * FIXME: For now, just ignore any alternatives which add retpolines.  This is
1131  * a temporary hack, as it doesn't allow ORC to unwind from inside a retpoline.
1132  * But it at least allows objtool to understand the control flow *around* the
1133  * retpoline.
1134  */
add_ignore_alternatives(struct objtool_file * file)1135 static int add_ignore_alternatives(struct objtool_file *file)
1136 {
1137 	struct section *sec;
1138 	struct reloc *reloc;
1139 	struct instruction *insn;
1140 
1141 	sec = find_section_by_name(file->elf, ".rela.discard.ignore_alts");
1142 	if (!sec)
1143 		return 0;
1144 
1145 	list_for_each_entry(reloc, &sec->reloc_list, list) {
1146 		if (reloc->sym->type != STT_SECTION) {
1147 			WARN("unexpected relocation symbol type in %s", sec->name);
1148 			return -1;
1149 		}
1150 
1151 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
1152 		if (!insn) {
1153 			WARN("bad .discard.ignore_alts entry");
1154 			return -1;
1155 		}
1156 
1157 		insn->ignore_alts = true;
1158 	}
1159 
1160 	return 0;
1161 }
1162 
arch_is_retpoline(struct symbol * sym)1163 __weak bool arch_is_retpoline(struct symbol *sym)
1164 {
1165 	return false;
1166 }
1167 
arch_is_rethunk(struct symbol * sym)1168 __weak bool arch_is_rethunk(struct symbol *sym)
1169 {
1170 	return false;
1171 }
1172 
1173 #define NEGATIVE_RELOC	((void *)-1L)
1174 
insn_reloc(struct objtool_file * file,struct instruction * insn)1175 static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn)
1176 {
1177 	if (insn->reloc == NEGATIVE_RELOC)
1178 		return NULL;
1179 
1180 	if (!insn->reloc) {
1181 		if (!file)
1182 			return NULL;
1183 
1184 		insn->reloc = find_reloc_by_dest_range(file->elf, insn->sec,
1185 						       insn->offset, insn->len);
1186 		if (!insn->reloc) {
1187 			insn->reloc = NEGATIVE_RELOC;
1188 			return NULL;
1189 		}
1190 	}
1191 
1192 	return insn->reloc;
1193 }
1194 
remove_insn_ops(struct instruction * insn)1195 static void remove_insn_ops(struct instruction *insn)
1196 {
1197 	struct stack_op *op, *tmp;
1198 
1199 	list_for_each_entry_safe(op, tmp, &insn->stack_ops, list) {
1200 		list_del(&op->list);
1201 		free(op);
1202 	}
1203 }
1204 
annotate_call_site(struct objtool_file * file,struct instruction * insn,bool sibling)1205 static void annotate_call_site(struct objtool_file *file,
1206 			       struct instruction *insn, bool sibling)
1207 {
1208 	struct reloc *reloc = insn_reloc(file, insn);
1209 	struct symbol *sym = insn->call_dest;
1210 
1211 	if (!sym)
1212 		sym = reloc->sym;
1213 
1214 	/*
1215 	 * Alternative replacement code is just template code which is
1216 	 * sometimes copied to the original instruction. For now, don't
1217 	 * annotate it. (In the future we might consider annotating the
1218 	 * original instruction if/when it ever makes sense to do so.)
1219 	 */
1220 	if (!strcmp(insn->sec->name, ".altinstr_replacement"))
1221 		return;
1222 
1223 	if (sym->static_call_tramp) {
1224 		list_add_tail(&insn->call_node, &file->static_call_list);
1225 		return;
1226 	}
1227 
1228 	if (sym->retpoline_thunk) {
1229 		list_add_tail(&insn->call_node, &file->retpoline_call_list);
1230 		return;
1231 	}
1232 
1233 	/*
1234 	 * Many compilers cannot disable KCOV or sanitizer calls with a function
1235 	 * attribute so they need a little help, NOP out any such calls from
1236 	 * noinstr text.
1237 	 */
1238 	if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) {
1239 		if (reloc) {
1240 			reloc->type = R_NONE;
1241 			elf_write_reloc(file->elf, reloc);
1242 		}
1243 
1244 		elf_write_insn(file->elf, insn->sec,
1245 			       insn->offset, insn->len,
1246 			       sibling ? arch_ret_insn(insn->len)
1247 			               : arch_nop_insn(insn->len));
1248 
1249 		insn->type = sibling ? INSN_RETURN : INSN_NOP;
1250 
1251 		if (sibling) {
1252 			/*
1253 			 * We've replaced the tail-call JMP insn by two new
1254 			 * insn: RET; INT3, except we only have a single struct
1255 			 * insn here. Mark it retpoline_safe to avoid the SLS
1256 			 * warning, instead of adding another insn.
1257 			 */
1258 			insn->retpoline_safe = true;
1259 		}
1260 
1261 		return;
1262 	}
1263 
1264 	if (opts.mcount && sym->fentry) {
1265 		if (sibling)
1266 			WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset);
1267 
1268 		if (reloc) {
1269 			reloc->type = R_NONE;
1270 			elf_write_reloc(file->elf, reloc);
1271 		}
1272 
1273 		elf_write_insn(file->elf, insn->sec,
1274 			       insn->offset, insn->len,
1275 			       arch_nop_insn(insn->len));
1276 
1277 		insn->type = INSN_NOP;
1278 
1279 		list_add_tail(&insn->call_node, &file->mcount_loc_list);
1280 		return;
1281 	}
1282 
1283 	if (!sibling && dead_end_function(file, sym))
1284 		insn->dead_end = true;
1285 }
1286 
add_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * dest,bool sibling)1287 static void add_call_dest(struct objtool_file *file, struct instruction *insn,
1288 			  struct symbol *dest, bool sibling)
1289 {
1290 	insn->call_dest = dest;
1291 	if (!dest)
1292 		return;
1293 
1294 	/*
1295 	 * Whatever stack impact regular CALLs have, should be undone
1296 	 * by the RETURN of the called function.
1297 	 *
1298 	 * Annotated intra-function calls retain the stack_ops but
1299 	 * are converted to JUMP, see read_intra_function_calls().
1300 	 */
1301 	remove_insn_ops(insn);
1302 
1303 	annotate_call_site(file, insn, sibling);
1304 }
1305 
add_retpoline_call(struct objtool_file * file,struct instruction * insn)1306 static void add_retpoline_call(struct objtool_file *file, struct instruction *insn)
1307 {
1308 	/*
1309 	 * Retpoline calls/jumps are really dynamic calls/jumps in disguise,
1310 	 * so convert them accordingly.
1311 	 */
1312 	switch (insn->type) {
1313 	case INSN_CALL:
1314 		insn->type = INSN_CALL_DYNAMIC;
1315 		break;
1316 	case INSN_JUMP_UNCONDITIONAL:
1317 		insn->type = INSN_JUMP_DYNAMIC;
1318 		break;
1319 	case INSN_JUMP_CONDITIONAL:
1320 		insn->type = INSN_JUMP_DYNAMIC_CONDITIONAL;
1321 		break;
1322 	default:
1323 		return;
1324 	}
1325 
1326 	insn->retpoline_safe = true;
1327 
1328 	/*
1329 	 * Whatever stack impact regular CALLs have, should be undone
1330 	 * by the RETURN of the called function.
1331 	 *
1332 	 * Annotated intra-function calls retain the stack_ops but
1333 	 * are converted to JUMP, see read_intra_function_calls().
1334 	 */
1335 	remove_insn_ops(insn);
1336 
1337 	annotate_call_site(file, insn, false);
1338 }
1339 
add_return_call(struct objtool_file * file,struct instruction * insn,bool add)1340 static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add)
1341 {
1342 	/*
1343 	 * Return thunk tail calls are really just returns in disguise,
1344 	 * so convert them accordingly.
1345 	 */
1346 	insn->type = INSN_RETURN;
1347 	insn->retpoline_safe = true;
1348 
1349 	if (add)
1350 		list_add_tail(&insn->call_node, &file->return_thunk_list);
1351 }
1352 
same_function(struct instruction * insn1,struct instruction * insn2)1353 static bool same_function(struct instruction *insn1, struct instruction *insn2)
1354 {
1355 	return insn1->func->pfunc == insn2->func->pfunc;
1356 }
1357 
is_first_func_insn(struct objtool_file * file,struct instruction * insn)1358 static bool is_first_func_insn(struct objtool_file *file, struct instruction *insn)
1359 {
1360 	if (insn->offset == insn->func->offset)
1361 		return true;
1362 
1363 	if (opts.ibt) {
1364 		struct instruction *prev = prev_insn_same_sym(file, insn);
1365 
1366 		if (prev && prev->type == INSN_ENDBR &&
1367 		    insn->offset == insn->func->offset + prev->len)
1368 			return true;
1369 	}
1370 
1371 	return false;
1372 }
1373 
1374 /*
1375  * Find the destination instructions for all jumps.
1376  */
add_jump_destinations(struct objtool_file * file)1377 static int add_jump_destinations(struct objtool_file *file)
1378 {
1379 	struct instruction *insn, *jump_dest;
1380 	struct reloc *reloc;
1381 	struct section *dest_sec;
1382 	unsigned long dest_off;
1383 
1384 	for_each_insn(file, insn) {
1385 		if (insn->jump_dest) {
1386 			/*
1387 			 * handle_group_alt() may have previously set
1388 			 * 'jump_dest' for some alternatives.
1389 			 */
1390 			continue;
1391 		}
1392 		if (!is_static_jump(insn))
1393 			continue;
1394 
1395 		reloc = insn_reloc(file, insn);
1396 		if (!reloc) {
1397 			dest_sec = insn->sec;
1398 			dest_off = arch_jump_destination(insn);
1399 		} else if (reloc->sym->type == STT_SECTION) {
1400 			dest_sec = reloc->sym->sec;
1401 			dest_off = arch_dest_reloc_offset(reloc->addend);
1402 		} else if (reloc->sym->retpoline_thunk) {
1403 			add_retpoline_call(file, insn);
1404 			continue;
1405 		} else if (reloc->sym->return_thunk) {
1406 			add_return_call(file, insn, true);
1407 			continue;
1408 		} else if (insn->func) {
1409 			/*
1410 			 * External sibling call or internal sibling call with
1411 			 * STT_FUNC reloc.
1412 			 */
1413 			add_call_dest(file, insn, reloc->sym, true);
1414 			continue;
1415 		} else if (reloc->sym->sec->idx) {
1416 			dest_sec = reloc->sym->sec;
1417 			dest_off = reloc->sym->sym.st_value +
1418 				   arch_dest_reloc_offset(reloc->addend);
1419 		} else {
1420 			/* non-func asm code jumping to another file */
1421 			continue;
1422 		}
1423 
1424 		jump_dest = find_insn(file, dest_sec, dest_off);
1425 		if (!jump_dest) {
1426 			struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off);
1427 
1428 			/*
1429 			 * This is a special case for zen_untrain_ret().
1430 			 * It jumps to __x86_return_thunk(), but objtool
1431 			 * can't find the thunk's starting RET
1432 			 * instruction, because the RET is also in the
1433 			 * middle of another instruction.  Objtool only
1434 			 * knows about the outer instruction.
1435 			 */
1436 			if (sym && sym->return_thunk) {
1437 				add_return_call(file, insn, false);
1438 				continue;
1439 			}
1440 
1441 			WARN_FUNC("can't find jump dest instruction at %s+0x%lx",
1442 				  insn->sec, insn->offset, dest_sec->name,
1443 				  dest_off);
1444 			return -1;
1445 		}
1446 
1447 		/*
1448 		 * Cross-function jump.
1449 		 */
1450 		if (insn->func && jump_dest->func &&
1451 		    insn->func != jump_dest->func) {
1452 
1453 			/*
1454 			 * For GCC 8+, create parent/child links for any cold
1455 			 * subfunctions.  This is _mostly_ redundant with a
1456 			 * similar initialization in read_symbols().
1457 			 *
1458 			 * If a function has aliases, we want the *first* such
1459 			 * function in the symbol table to be the subfunction's
1460 			 * parent.  In that case we overwrite the
1461 			 * initialization done in read_symbols().
1462 			 *
1463 			 * However this code can't completely replace the
1464 			 * read_symbols() code because this doesn't detect the
1465 			 * case where the parent function's only reference to a
1466 			 * subfunction is through a jump table.
1467 			 */
1468 			if (!strstr(insn->func->name, ".cold") &&
1469 			    strstr(jump_dest->func->name, ".cold")) {
1470 				insn->func->cfunc = jump_dest->func;
1471 				jump_dest->func->pfunc = insn->func;
1472 
1473 			} else if (!same_function(insn, jump_dest) &&
1474 				   is_first_func_insn(file, jump_dest)) {
1475 				/*
1476 				 * Internal sibling call without reloc or with
1477 				 * STT_SECTION reloc.
1478 				 */
1479 				add_call_dest(file, insn, jump_dest->func, true);
1480 				continue;
1481 			}
1482 		}
1483 
1484 		insn->jump_dest = jump_dest;
1485 	}
1486 
1487 	return 0;
1488 }
1489 
find_call_destination(struct section * sec,unsigned long offset)1490 static struct symbol *find_call_destination(struct section *sec, unsigned long offset)
1491 {
1492 	struct symbol *call_dest;
1493 
1494 	call_dest = find_func_by_offset(sec, offset);
1495 	if (!call_dest)
1496 		call_dest = find_symbol_by_offset(sec, offset);
1497 
1498 	return call_dest;
1499 }
1500 
1501 /*
1502  * Find the destination instructions for all calls.
1503  */
add_call_destinations(struct objtool_file * file)1504 static int add_call_destinations(struct objtool_file *file)
1505 {
1506 	struct instruction *insn;
1507 	unsigned long dest_off;
1508 	struct symbol *dest;
1509 	struct reloc *reloc;
1510 
1511 	for_each_insn(file, insn) {
1512 		if (insn->type != INSN_CALL)
1513 			continue;
1514 
1515 		reloc = insn_reloc(file, insn);
1516 		if (!reloc) {
1517 			dest_off = arch_jump_destination(insn);
1518 			dest = find_call_destination(insn->sec, dest_off);
1519 
1520 			add_call_dest(file, insn, dest, false);
1521 
1522 			if (insn->ignore)
1523 				continue;
1524 
1525 			if (!insn->call_dest) {
1526 				WARN_FUNC("unannotated intra-function call", insn->sec, insn->offset);
1527 				return -1;
1528 			}
1529 
1530 			if (insn->func && insn->call_dest->type != STT_FUNC) {
1531 				WARN_FUNC("unsupported call to non-function",
1532 					  insn->sec, insn->offset);
1533 				return -1;
1534 			}
1535 
1536 		} else if (reloc->sym->type == STT_SECTION) {
1537 			dest_off = arch_dest_reloc_offset(reloc->addend);
1538 			dest = find_call_destination(reloc->sym->sec, dest_off);
1539 			if (!dest) {
1540 				WARN_FUNC("can't find call dest symbol at %s+0x%lx",
1541 					  insn->sec, insn->offset,
1542 					  reloc->sym->sec->name,
1543 					  dest_off);
1544 				return -1;
1545 			}
1546 
1547 			add_call_dest(file, insn, dest, false);
1548 
1549 		} else if (reloc->sym->retpoline_thunk) {
1550 			add_retpoline_call(file, insn);
1551 
1552 		} else
1553 			add_call_dest(file, insn, reloc->sym, false);
1554 	}
1555 
1556 	return 0;
1557 }
1558 
1559 /*
1560  * The .alternatives section requires some extra special care over and above
1561  * other special sections because alternatives are patched in place.
1562  */
handle_group_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1563 static int handle_group_alt(struct objtool_file *file,
1564 			    struct special_alt *special_alt,
1565 			    struct instruction *orig_insn,
1566 			    struct instruction **new_insn)
1567 {
1568 	struct instruction *last_orig_insn, *last_new_insn = NULL, *insn, *nop = NULL;
1569 	struct alt_group *orig_alt_group, *new_alt_group;
1570 	unsigned long dest_off;
1571 
1572 
1573 	orig_alt_group = malloc(sizeof(*orig_alt_group));
1574 	if (!orig_alt_group) {
1575 		WARN("malloc failed");
1576 		return -1;
1577 	}
1578 	orig_alt_group->cfi = calloc(special_alt->orig_len,
1579 				     sizeof(struct cfi_state *));
1580 	if (!orig_alt_group->cfi) {
1581 		WARN("calloc failed");
1582 		return -1;
1583 	}
1584 
1585 	last_orig_insn = NULL;
1586 	insn = orig_insn;
1587 	sec_for_each_insn_from(file, insn) {
1588 		if (insn->offset >= special_alt->orig_off + special_alt->orig_len)
1589 			break;
1590 
1591 		insn->alt_group = orig_alt_group;
1592 		last_orig_insn = insn;
1593 	}
1594 	orig_alt_group->orig_group = NULL;
1595 	orig_alt_group->first_insn = orig_insn;
1596 	orig_alt_group->last_insn = last_orig_insn;
1597 
1598 
1599 	new_alt_group = malloc(sizeof(*new_alt_group));
1600 	if (!new_alt_group) {
1601 		WARN("malloc failed");
1602 		return -1;
1603 	}
1604 
1605 	if (special_alt->new_len < special_alt->orig_len) {
1606 		/*
1607 		 * Insert a fake nop at the end to make the replacement
1608 		 * alt_group the same size as the original.  This is needed to
1609 		 * allow propagate_alt_cfi() to do its magic.  When the last
1610 		 * instruction affects the stack, the instruction after it (the
1611 		 * nop) will propagate the new state to the shared CFI array.
1612 		 */
1613 		nop = malloc(sizeof(*nop));
1614 		if (!nop) {
1615 			WARN("malloc failed");
1616 			return -1;
1617 		}
1618 		memset(nop, 0, sizeof(*nop));
1619 		INIT_LIST_HEAD(&nop->alts);
1620 		INIT_LIST_HEAD(&nop->stack_ops);
1621 
1622 		nop->sec = special_alt->new_sec;
1623 		nop->offset = special_alt->new_off + special_alt->new_len;
1624 		nop->len = special_alt->orig_len - special_alt->new_len;
1625 		nop->type = INSN_NOP;
1626 		nop->func = orig_insn->func;
1627 		nop->alt_group = new_alt_group;
1628 		nop->ignore = orig_insn->ignore_alts;
1629 	}
1630 
1631 	if (!special_alt->new_len) {
1632 		*new_insn = nop;
1633 		goto end;
1634 	}
1635 
1636 	insn = *new_insn;
1637 	sec_for_each_insn_from(file, insn) {
1638 		struct reloc *alt_reloc;
1639 
1640 		if (insn->offset >= special_alt->new_off + special_alt->new_len)
1641 			break;
1642 
1643 		last_new_insn = insn;
1644 
1645 		insn->ignore = orig_insn->ignore_alts;
1646 		insn->func = orig_insn->func;
1647 		insn->alt_group = new_alt_group;
1648 
1649 		/*
1650 		 * Since alternative replacement code is copy/pasted by the
1651 		 * kernel after applying relocations, generally such code can't
1652 		 * have relative-address relocation references to outside the
1653 		 * .altinstr_replacement section, unless the arch's
1654 		 * alternatives code can adjust the relative offsets
1655 		 * accordingly.
1656 		 */
1657 		alt_reloc = insn_reloc(file, insn);
1658 		if (alt_reloc &&
1659 		    !arch_support_alt_relocation(special_alt, insn, alt_reloc)) {
1660 
1661 			WARN_FUNC("unsupported relocation in alternatives section",
1662 				  insn->sec, insn->offset);
1663 			return -1;
1664 		}
1665 
1666 		if (!is_static_jump(insn))
1667 			continue;
1668 
1669 		if (!insn->immediate)
1670 			continue;
1671 
1672 		dest_off = arch_jump_destination(insn);
1673 		if (dest_off == special_alt->new_off + special_alt->new_len) {
1674 			insn->jump_dest = next_insn_same_sec(file, last_orig_insn);
1675 			if (!insn->jump_dest) {
1676 				WARN_FUNC("can't find alternative jump destination",
1677 					  insn->sec, insn->offset);
1678 				return -1;
1679 			}
1680 		}
1681 	}
1682 
1683 	if (!last_new_insn) {
1684 		WARN_FUNC("can't find last new alternative instruction",
1685 			  special_alt->new_sec, special_alt->new_off);
1686 		return -1;
1687 	}
1688 
1689 	if (nop)
1690 		list_add(&nop->list, &last_new_insn->list);
1691 end:
1692 	new_alt_group->orig_group = orig_alt_group;
1693 	new_alt_group->first_insn = *new_insn;
1694 	new_alt_group->last_insn = nop ? : last_new_insn;
1695 	new_alt_group->cfi = orig_alt_group->cfi;
1696 	return 0;
1697 }
1698 
1699 /*
1700  * A jump table entry can either convert a nop to a jump or a jump to a nop.
1701  * If the original instruction is a jump, make the alt entry an effective nop
1702  * by just skipping the original instruction.
1703  */
handle_jump_alt(struct objtool_file * file,struct special_alt * special_alt,struct instruction * orig_insn,struct instruction ** new_insn)1704 static int handle_jump_alt(struct objtool_file *file,
1705 			   struct special_alt *special_alt,
1706 			   struct instruction *orig_insn,
1707 			   struct instruction **new_insn)
1708 {
1709 	if (orig_insn->type != INSN_JUMP_UNCONDITIONAL &&
1710 	    orig_insn->type != INSN_NOP) {
1711 
1712 		WARN_FUNC("unsupported instruction at jump label",
1713 			  orig_insn->sec, orig_insn->offset);
1714 		return -1;
1715 	}
1716 
1717 	if (opts.hack_jump_label && special_alt->key_addend & 2) {
1718 		struct reloc *reloc = insn_reloc(file, orig_insn);
1719 
1720 		if (reloc) {
1721 			reloc->type = R_NONE;
1722 			elf_write_reloc(file->elf, reloc);
1723 		}
1724 		elf_write_insn(file->elf, orig_insn->sec,
1725 			       orig_insn->offset, orig_insn->len,
1726 			       arch_nop_insn(orig_insn->len));
1727 		orig_insn->type = INSN_NOP;
1728 	}
1729 
1730 	if (orig_insn->type == INSN_NOP) {
1731 		if (orig_insn->len == 2)
1732 			file->jl_nop_short++;
1733 		else
1734 			file->jl_nop_long++;
1735 
1736 		return 0;
1737 	}
1738 
1739 	if (orig_insn->len == 2)
1740 		file->jl_short++;
1741 	else
1742 		file->jl_long++;
1743 
1744 	*new_insn = list_next_entry(orig_insn, list);
1745 	return 0;
1746 }
1747 
1748 /*
1749  * Read all the special sections which have alternate instructions which can be
1750  * patched in or redirected to at runtime.  Each instruction having alternate
1751  * instruction(s) has them added to its insn->alts list, which will be
1752  * traversed in validate_branch().
1753  */
add_special_section_alts(struct objtool_file * file)1754 static int add_special_section_alts(struct objtool_file *file)
1755 {
1756 	struct list_head special_alts;
1757 	struct instruction *orig_insn, *new_insn;
1758 	struct special_alt *special_alt, *tmp;
1759 	struct alternative *alt;
1760 	int ret;
1761 
1762 	ret = special_get_alts(file->elf, &special_alts);
1763 	if (ret)
1764 		return ret;
1765 
1766 	list_for_each_entry_safe(special_alt, tmp, &special_alts, list) {
1767 
1768 		orig_insn = find_insn(file, special_alt->orig_sec,
1769 				      special_alt->orig_off);
1770 		if (!orig_insn) {
1771 			WARN_FUNC("special: can't find orig instruction",
1772 				  special_alt->orig_sec, special_alt->orig_off);
1773 			ret = -1;
1774 			goto out;
1775 		}
1776 
1777 		new_insn = NULL;
1778 		if (!special_alt->group || special_alt->new_len) {
1779 			new_insn = find_insn(file, special_alt->new_sec,
1780 					     special_alt->new_off);
1781 			if (!new_insn) {
1782 				WARN_FUNC("special: can't find new instruction",
1783 					  special_alt->new_sec,
1784 					  special_alt->new_off);
1785 				ret = -1;
1786 				goto out;
1787 			}
1788 		}
1789 
1790 		if (special_alt->group) {
1791 			if (!special_alt->orig_len) {
1792 				WARN_FUNC("empty alternative entry",
1793 					  orig_insn->sec, orig_insn->offset);
1794 				continue;
1795 			}
1796 
1797 			ret = handle_group_alt(file, special_alt, orig_insn,
1798 					       &new_insn);
1799 			if (ret)
1800 				goto out;
1801 		} else if (special_alt->jump_or_nop) {
1802 			ret = handle_jump_alt(file, special_alt, orig_insn,
1803 					      &new_insn);
1804 			if (ret)
1805 				goto out;
1806 		}
1807 
1808 		alt = malloc(sizeof(*alt));
1809 		if (!alt) {
1810 			WARN("malloc failed");
1811 			ret = -1;
1812 			goto out;
1813 		}
1814 
1815 		alt->insn = new_insn;
1816 		alt->skip_orig = special_alt->skip_orig;
1817 		orig_insn->ignore_alts |= special_alt->skip_alt;
1818 		list_add_tail(&alt->list, &orig_insn->alts);
1819 
1820 		list_del(&special_alt->list);
1821 		free(special_alt);
1822 	}
1823 
1824 	if (opts.stats) {
1825 		printf("jl\\\tNOP\tJMP\n");
1826 		printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short);
1827 		printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long);
1828 	}
1829 
1830 out:
1831 	return ret;
1832 }
1833 
add_jump_table(struct objtool_file * file,struct instruction * insn,struct reloc * table)1834 static int add_jump_table(struct objtool_file *file, struct instruction *insn,
1835 			    struct reloc *table)
1836 {
1837 	struct reloc *reloc = table;
1838 	struct instruction *dest_insn;
1839 	struct alternative *alt;
1840 	struct symbol *pfunc = insn->func->pfunc;
1841 	unsigned int prev_offset = 0;
1842 
1843 	/*
1844 	 * Each @reloc is a switch table relocation which points to the target
1845 	 * instruction.
1846 	 */
1847 	list_for_each_entry_from(reloc, &table->sec->reloc_list, list) {
1848 
1849 		/* Check for the end of the table: */
1850 		if (reloc != table && reloc->jump_table_start)
1851 			break;
1852 
1853 		/* Make sure the table entries are consecutive: */
1854 		if (prev_offset && reloc->offset != prev_offset + 8)
1855 			break;
1856 
1857 		/* Detect function pointers from contiguous objects: */
1858 		if (reloc->sym->sec == pfunc->sec &&
1859 		    reloc->addend == pfunc->offset)
1860 			break;
1861 
1862 		dest_insn = find_insn(file, reloc->sym->sec, reloc->addend);
1863 		if (!dest_insn)
1864 			break;
1865 
1866 		/* Make sure the destination is in the same function: */
1867 		if (!dest_insn->func || dest_insn->func->pfunc != pfunc)
1868 			break;
1869 
1870 		alt = malloc(sizeof(*alt));
1871 		if (!alt) {
1872 			WARN("malloc failed");
1873 			return -1;
1874 		}
1875 
1876 		alt->insn = dest_insn;
1877 		list_add_tail(&alt->list, &insn->alts);
1878 		prev_offset = reloc->offset;
1879 	}
1880 
1881 	if (!prev_offset) {
1882 		WARN_FUNC("can't find switch jump table",
1883 			  insn->sec, insn->offset);
1884 		return -1;
1885 	}
1886 
1887 	return 0;
1888 }
1889 
1890 /*
1891  * find_jump_table() - Given a dynamic jump, find the switch jump table
1892  * associated with it.
1893  */
find_jump_table(struct objtool_file * file,struct symbol * func,struct instruction * insn)1894 static struct reloc *find_jump_table(struct objtool_file *file,
1895 				      struct symbol *func,
1896 				      struct instruction *insn)
1897 {
1898 	struct reloc *table_reloc;
1899 	struct instruction *dest_insn, *orig_insn = insn;
1900 
1901 	/*
1902 	 * Backward search using the @first_jump_src links, these help avoid
1903 	 * much of the 'in between' code. Which avoids us getting confused by
1904 	 * it.
1905 	 */
1906 	for (;
1907 	     insn && insn->func && insn->func->pfunc == func;
1908 	     insn = insn->first_jump_src ?: prev_insn_same_sym(file, insn)) {
1909 
1910 		if (insn != orig_insn && insn->type == INSN_JUMP_DYNAMIC)
1911 			break;
1912 
1913 		/* allow small jumps within the range */
1914 		if (insn->type == INSN_JUMP_UNCONDITIONAL &&
1915 		    insn->jump_dest &&
1916 		    (insn->jump_dest->offset <= insn->offset ||
1917 		     insn->jump_dest->offset > orig_insn->offset))
1918 		    break;
1919 
1920 		table_reloc = arch_find_switch_table(file, insn);
1921 		if (!table_reloc)
1922 			continue;
1923 		dest_insn = find_insn(file, table_reloc->sym->sec, table_reloc->addend);
1924 		if (!dest_insn || !dest_insn->func || dest_insn->func->pfunc != func)
1925 			continue;
1926 
1927 		return table_reloc;
1928 	}
1929 
1930 	return NULL;
1931 }
1932 
1933 /*
1934  * First pass: Mark the head of each jump table so that in the next pass,
1935  * we know when a given jump table ends and the next one starts.
1936  */
mark_func_jump_tables(struct objtool_file * file,struct symbol * func)1937 static void mark_func_jump_tables(struct objtool_file *file,
1938 				    struct symbol *func)
1939 {
1940 	struct instruction *insn, *last = NULL;
1941 	struct reloc *reloc;
1942 
1943 	func_for_each_insn(file, func, insn) {
1944 		if (!last)
1945 			last = insn;
1946 
1947 		/*
1948 		 * Store back-pointers for unconditional forward jumps such
1949 		 * that find_jump_table() can back-track using those and
1950 		 * avoid some potentially confusing code.
1951 		 */
1952 		if (insn->type == INSN_JUMP_UNCONDITIONAL && insn->jump_dest &&
1953 		    insn->offset > last->offset &&
1954 		    insn->jump_dest->offset > insn->offset &&
1955 		    !insn->jump_dest->first_jump_src) {
1956 
1957 			insn->jump_dest->first_jump_src = insn;
1958 			last = insn->jump_dest;
1959 		}
1960 
1961 		if (insn->type != INSN_JUMP_DYNAMIC)
1962 			continue;
1963 
1964 		reloc = find_jump_table(file, func, insn);
1965 		if (reloc) {
1966 			reloc->jump_table_start = true;
1967 			insn->jump_table = reloc;
1968 		}
1969 	}
1970 }
1971 
add_func_jump_tables(struct objtool_file * file,struct symbol * func)1972 static int add_func_jump_tables(struct objtool_file *file,
1973 				  struct symbol *func)
1974 {
1975 	struct instruction *insn;
1976 	int ret;
1977 
1978 	func_for_each_insn(file, func, insn) {
1979 		if (!insn->jump_table)
1980 			continue;
1981 
1982 		ret = add_jump_table(file, insn, insn->jump_table);
1983 		if (ret)
1984 			return ret;
1985 	}
1986 
1987 	return 0;
1988 }
1989 
1990 /*
1991  * For some switch statements, gcc generates a jump table in the .rodata
1992  * section which contains a list of addresses within the function to jump to.
1993  * This finds these jump tables and adds them to the insn->alts lists.
1994  */
add_jump_table_alts(struct objtool_file * file)1995 static int add_jump_table_alts(struct objtool_file *file)
1996 {
1997 	struct section *sec;
1998 	struct symbol *func;
1999 	int ret;
2000 
2001 	if (!file->rodata)
2002 		return 0;
2003 
2004 	for_each_sec(file, sec) {
2005 		list_for_each_entry(func, &sec->symbol_list, list) {
2006 			if (func->type != STT_FUNC)
2007 				continue;
2008 
2009 			mark_func_jump_tables(file, func);
2010 			ret = add_func_jump_tables(file, func);
2011 			if (ret)
2012 				return ret;
2013 		}
2014 	}
2015 
2016 	return 0;
2017 }
2018 
set_func_state(struct cfi_state * state)2019 static void set_func_state(struct cfi_state *state)
2020 {
2021 	state->cfa = initial_func_cfi.cfa;
2022 	memcpy(&state->regs, &initial_func_cfi.regs,
2023 	       CFI_NUM_REGS * sizeof(struct cfi_reg));
2024 	state->stack_size = initial_func_cfi.cfa.offset;
2025 }
2026 
read_unwind_hints(struct objtool_file * file)2027 static int read_unwind_hints(struct objtool_file *file)
2028 {
2029 	struct cfi_state cfi = init_cfi;
2030 	struct section *sec, *relocsec;
2031 	struct unwind_hint *hint;
2032 	struct instruction *insn;
2033 	struct reloc *reloc;
2034 	int i;
2035 
2036 	sec = find_section_by_name(file->elf, ".discard.unwind_hints");
2037 	if (!sec)
2038 		return 0;
2039 
2040 	relocsec = sec->reloc;
2041 	if (!relocsec) {
2042 		WARN("missing .rela.discard.unwind_hints section");
2043 		return -1;
2044 	}
2045 
2046 	if (sec->sh.sh_size % sizeof(struct unwind_hint)) {
2047 		WARN("struct unwind_hint size mismatch");
2048 		return -1;
2049 	}
2050 
2051 	file->hints = true;
2052 
2053 	for (i = 0; i < sec->sh.sh_size / sizeof(struct unwind_hint); i++) {
2054 		hint = (struct unwind_hint *)sec->data->d_buf + i;
2055 
2056 		reloc = find_reloc_by_dest(file->elf, sec, i * sizeof(*hint));
2057 		if (!reloc) {
2058 			WARN("can't find reloc for unwind_hints[%d]", i);
2059 			return -1;
2060 		}
2061 
2062 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2063 		if (!insn) {
2064 			WARN("can't find insn for unwind_hints[%d]", i);
2065 			return -1;
2066 		}
2067 
2068 		insn->hint = true;
2069 
2070 		if (hint->type == UNWIND_HINT_TYPE_SAVE) {
2071 			insn->hint = false;
2072 			insn->save = true;
2073 			continue;
2074 		}
2075 
2076 		if (hint->type == UNWIND_HINT_TYPE_RESTORE) {
2077 			insn->restore = true;
2078 			continue;
2079 		}
2080 
2081 		if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) {
2082 			struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset);
2083 
2084 			if (sym && sym->bind == STB_GLOBAL) {
2085 				if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) {
2086 					WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR",
2087 						  insn->sec, insn->offset);
2088 				}
2089 
2090 				insn->entry = 1;
2091 			}
2092 		}
2093 
2094 		if (hint->type == UNWIND_HINT_TYPE_ENTRY) {
2095 			hint->type = UNWIND_HINT_TYPE_CALL;
2096 			insn->entry = 1;
2097 		}
2098 
2099 		if (hint->type == UNWIND_HINT_TYPE_FUNC) {
2100 			insn->cfi = &func_cfi;
2101 			continue;
2102 		}
2103 
2104 		if (insn->cfi)
2105 			cfi = *(insn->cfi);
2106 
2107 		if (arch_decode_hint_reg(hint->sp_reg, &cfi.cfa.base)) {
2108 			WARN_FUNC("unsupported unwind_hint sp base reg %d",
2109 				  insn->sec, insn->offset, hint->sp_reg);
2110 			return -1;
2111 		}
2112 
2113 		cfi.cfa.offset = bswap_if_needed(hint->sp_offset);
2114 		cfi.type = hint->type;
2115 		cfi.end = hint->end;
2116 
2117 		insn->cfi = cfi_hash_find_or_add(&cfi);
2118 	}
2119 
2120 	return 0;
2121 }
2122 
read_noendbr_hints(struct objtool_file * file)2123 static int read_noendbr_hints(struct objtool_file *file)
2124 {
2125 	struct section *sec;
2126 	struct instruction *insn;
2127 	struct reloc *reloc;
2128 
2129 	sec = find_section_by_name(file->elf, ".rela.discard.noendbr");
2130 	if (!sec)
2131 		return 0;
2132 
2133 	list_for_each_entry(reloc, &sec->reloc_list, list) {
2134 		insn = find_insn(file, reloc->sym->sec, reloc->sym->offset + reloc->addend);
2135 		if (!insn) {
2136 			WARN("bad .discard.noendbr entry");
2137 			return -1;
2138 		}
2139 
2140 		insn->noendbr = 1;
2141 	}
2142 
2143 	return 0;
2144 }
2145 
read_retpoline_hints(struct objtool_file * file)2146 static int read_retpoline_hints(struct objtool_file *file)
2147 {
2148 	struct section *sec;
2149 	struct instruction *insn;
2150 	struct reloc *reloc;
2151 
2152 	sec = find_section_by_name(file->elf, ".rela.discard.retpoline_safe");
2153 	if (!sec)
2154 		return 0;
2155 
2156 	list_for_each_entry(reloc, &sec->reloc_list, list) {
2157 		if (reloc->sym->type != STT_SECTION) {
2158 			WARN("unexpected relocation symbol type in %s", sec->name);
2159 			return -1;
2160 		}
2161 
2162 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2163 		if (!insn) {
2164 			WARN("bad .discard.retpoline_safe entry");
2165 			return -1;
2166 		}
2167 
2168 		if (insn->type != INSN_JUMP_DYNAMIC &&
2169 		    insn->type != INSN_CALL_DYNAMIC &&
2170 		    insn->type != INSN_RETURN &&
2171 		    insn->type != INSN_NOP) {
2172 			WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop",
2173 				  insn->sec, insn->offset);
2174 			return -1;
2175 		}
2176 
2177 		insn->retpoline_safe = true;
2178 	}
2179 
2180 	return 0;
2181 }
2182 
read_instr_hints(struct objtool_file * file)2183 static int read_instr_hints(struct objtool_file *file)
2184 {
2185 	struct section *sec;
2186 	struct instruction *insn;
2187 	struct reloc *reloc;
2188 
2189 	sec = find_section_by_name(file->elf, ".rela.discard.instr_end");
2190 	if (!sec)
2191 		return 0;
2192 
2193 	list_for_each_entry(reloc, &sec->reloc_list, list) {
2194 		if (reloc->sym->type != STT_SECTION) {
2195 			WARN("unexpected relocation symbol type in %s", sec->name);
2196 			return -1;
2197 		}
2198 
2199 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2200 		if (!insn) {
2201 			WARN("bad .discard.instr_end entry");
2202 			return -1;
2203 		}
2204 
2205 		insn->instr--;
2206 	}
2207 
2208 	sec = find_section_by_name(file->elf, ".rela.discard.instr_begin");
2209 	if (!sec)
2210 		return 0;
2211 
2212 	list_for_each_entry(reloc, &sec->reloc_list, list) {
2213 		if (reloc->sym->type != STT_SECTION) {
2214 			WARN("unexpected relocation symbol type in %s", sec->name);
2215 			return -1;
2216 		}
2217 
2218 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2219 		if (!insn) {
2220 			WARN("bad .discard.instr_begin entry");
2221 			return -1;
2222 		}
2223 
2224 		insn->instr++;
2225 	}
2226 
2227 	return 0;
2228 }
2229 
read_intra_function_calls(struct objtool_file * file)2230 static int read_intra_function_calls(struct objtool_file *file)
2231 {
2232 	struct instruction *insn;
2233 	struct section *sec;
2234 	struct reloc *reloc;
2235 
2236 	sec = find_section_by_name(file->elf, ".rela.discard.intra_function_calls");
2237 	if (!sec)
2238 		return 0;
2239 
2240 	list_for_each_entry(reloc, &sec->reloc_list, list) {
2241 		unsigned long dest_off;
2242 
2243 		if (reloc->sym->type != STT_SECTION) {
2244 			WARN("unexpected relocation symbol type in %s",
2245 			     sec->name);
2246 			return -1;
2247 		}
2248 
2249 		insn = find_insn(file, reloc->sym->sec, reloc->addend);
2250 		if (!insn) {
2251 			WARN("bad .discard.intra_function_call entry");
2252 			return -1;
2253 		}
2254 
2255 		if (insn->type != INSN_CALL) {
2256 			WARN_FUNC("intra_function_call not a direct call",
2257 				  insn->sec, insn->offset);
2258 			return -1;
2259 		}
2260 
2261 		/*
2262 		 * Treat intra-function CALLs as JMPs, but with a stack_op.
2263 		 * See add_call_destinations(), which strips stack_ops from
2264 		 * normal CALLs.
2265 		 */
2266 		insn->type = INSN_JUMP_UNCONDITIONAL;
2267 
2268 		dest_off = arch_jump_destination(insn);
2269 		insn->jump_dest = find_insn(file, insn->sec, dest_off);
2270 		if (!insn->jump_dest) {
2271 			WARN_FUNC("can't find call dest at %s+0x%lx",
2272 				  insn->sec, insn->offset,
2273 				  insn->sec->name, dest_off);
2274 			return -1;
2275 		}
2276 	}
2277 
2278 	return 0;
2279 }
2280 
2281 /*
2282  * Return true if name matches an instrumentation function, where calls to that
2283  * function from noinstr code can safely be removed, but compilers won't do so.
2284  */
is_profiling_func(const char * name)2285 static bool is_profiling_func(const char *name)
2286 {
2287 	/*
2288 	 * Many compilers cannot disable KCOV with a function attribute.
2289 	 */
2290 	if (!strncmp(name, "__sanitizer_cov_", 16))
2291 		return true;
2292 
2293 	/*
2294 	 * Some compilers currently do not remove __tsan_func_entry/exit nor
2295 	 * __tsan_atomic_signal_fence (used for barrier instrumentation) with
2296 	 * the __no_sanitize_thread attribute, remove them. Once the kernel's
2297 	 * minimum Clang version is 14.0, this can be removed.
2298 	 */
2299 	if (!strncmp(name, "__tsan_func_", 12) ||
2300 	    !strcmp(name, "__tsan_atomic_signal_fence"))
2301 		return true;
2302 
2303 	return false;
2304 }
2305 
classify_symbols(struct objtool_file * file)2306 static int classify_symbols(struct objtool_file *file)
2307 {
2308 	struct section *sec;
2309 	struct symbol *func;
2310 
2311 	for_each_sec(file, sec) {
2312 		list_for_each_entry(func, &sec->symbol_list, list) {
2313 			if (func->bind != STB_GLOBAL)
2314 				continue;
2315 
2316 			if (!strncmp(func->name, STATIC_CALL_TRAMP_PREFIX_STR,
2317 				     strlen(STATIC_CALL_TRAMP_PREFIX_STR)))
2318 				func->static_call_tramp = true;
2319 
2320 			if (arch_is_retpoline(func))
2321 				func->retpoline_thunk = true;
2322 
2323 			if (arch_is_rethunk(func))
2324 				func->return_thunk = true;
2325 
2326 			if (!strcmp(func->name, "__fentry__"))
2327 				func->fentry = true;
2328 
2329 			if (is_profiling_func(func->name))
2330 				func->profiling_func = true;
2331 		}
2332 	}
2333 
2334 	return 0;
2335 }
2336 
mark_rodata(struct objtool_file * file)2337 static void mark_rodata(struct objtool_file *file)
2338 {
2339 	struct section *sec;
2340 	bool found = false;
2341 
2342 	/*
2343 	 * Search for the following rodata sections, each of which can
2344 	 * potentially contain jump tables:
2345 	 *
2346 	 * - .rodata: can contain GCC switch tables
2347 	 * - .rodata.<func>: same, if -fdata-sections is being used
2348 	 * - .rodata..c_jump_table: contains C annotated jump tables
2349 	 *
2350 	 * .rodata.str1.* sections are ignored; they don't contain jump tables.
2351 	 */
2352 	for_each_sec(file, sec) {
2353 		if (!strncmp(sec->name, ".rodata", 7) &&
2354 		    !strstr(sec->name, ".str1.")) {
2355 			sec->rodata = true;
2356 			found = true;
2357 		}
2358 	}
2359 
2360 	file->rodata = found;
2361 }
2362 
decode_sections(struct objtool_file * file)2363 static int decode_sections(struct objtool_file *file)
2364 {
2365 	int ret;
2366 
2367 	mark_rodata(file);
2368 
2369 	ret = init_pv_ops(file);
2370 	if (ret)
2371 		return ret;
2372 
2373 	ret = decode_instructions(file);
2374 	if (ret)
2375 		return ret;
2376 
2377 	add_ignores(file);
2378 	add_uaccess_safe(file);
2379 
2380 	ret = add_ignore_alternatives(file);
2381 	if (ret)
2382 		return ret;
2383 
2384 	/*
2385 	 * Must be before read_unwind_hints() since that needs insn->noendbr.
2386 	 */
2387 	ret = read_noendbr_hints(file);
2388 	if (ret)
2389 		return ret;
2390 
2391 	/*
2392 	 * Must be before add_{jump_call}_destination.
2393 	 */
2394 	ret = classify_symbols(file);
2395 	if (ret)
2396 		return ret;
2397 
2398 	/*
2399 	 * Must be before add_jump_destinations(), which depends on 'func'
2400 	 * being set for alternatives, to enable proper sibling call detection.
2401 	 */
2402 	ret = add_special_section_alts(file);
2403 	if (ret)
2404 		return ret;
2405 
2406 	ret = add_jump_destinations(file);
2407 	if (ret)
2408 		return ret;
2409 
2410 	/*
2411 	 * Must be before add_call_destination(); it changes INSN_CALL to
2412 	 * INSN_JUMP.
2413 	 */
2414 	ret = read_intra_function_calls(file);
2415 	if (ret)
2416 		return ret;
2417 
2418 	ret = add_call_destinations(file);
2419 	if (ret)
2420 		return ret;
2421 
2422 	/*
2423 	 * Must be after add_call_destinations() such that it can override
2424 	 * dead_end_function() marks.
2425 	 */
2426 	ret = add_dead_ends(file);
2427 	if (ret)
2428 		return ret;
2429 
2430 	ret = add_jump_table_alts(file);
2431 	if (ret)
2432 		return ret;
2433 
2434 	ret = read_unwind_hints(file);
2435 	if (ret)
2436 		return ret;
2437 
2438 	ret = read_retpoline_hints(file);
2439 	if (ret)
2440 		return ret;
2441 
2442 	ret = read_instr_hints(file);
2443 	if (ret)
2444 		return ret;
2445 
2446 	return 0;
2447 }
2448 
is_fentry_call(struct instruction * insn)2449 static bool is_fentry_call(struct instruction *insn)
2450 {
2451 	if (insn->type == INSN_CALL &&
2452 	    insn->call_dest &&
2453 	    insn->call_dest->fentry)
2454 		return true;
2455 
2456 	return false;
2457 }
2458 
has_modified_stack_frame(struct instruction * insn,struct insn_state * state)2459 static bool has_modified_stack_frame(struct instruction *insn, struct insn_state *state)
2460 {
2461 	struct cfi_state *cfi = &state->cfi;
2462 	int i;
2463 
2464 	if (cfi->cfa.base != initial_func_cfi.cfa.base || cfi->drap)
2465 		return true;
2466 
2467 	if (cfi->cfa.offset != initial_func_cfi.cfa.offset)
2468 		return true;
2469 
2470 	if (cfi->stack_size != initial_func_cfi.cfa.offset)
2471 		return true;
2472 
2473 	for (i = 0; i < CFI_NUM_REGS; i++) {
2474 		if (cfi->regs[i].base != initial_func_cfi.regs[i].base ||
2475 		    cfi->regs[i].offset != initial_func_cfi.regs[i].offset)
2476 			return true;
2477 	}
2478 
2479 	return false;
2480 }
2481 
check_reg_frame_pos(const struct cfi_reg * reg,int expected_offset)2482 static bool check_reg_frame_pos(const struct cfi_reg *reg,
2483 				int expected_offset)
2484 {
2485 	return reg->base == CFI_CFA &&
2486 	       reg->offset == expected_offset;
2487 }
2488 
has_valid_stack_frame(struct insn_state * state)2489 static bool has_valid_stack_frame(struct insn_state *state)
2490 {
2491 	struct cfi_state *cfi = &state->cfi;
2492 
2493 	if (cfi->cfa.base == CFI_BP &&
2494 	    check_reg_frame_pos(&cfi->regs[CFI_BP], -cfi->cfa.offset) &&
2495 	    check_reg_frame_pos(&cfi->regs[CFI_RA], -cfi->cfa.offset + 8))
2496 		return true;
2497 
2498 	if (cfi->drap && cfi->regs[CFI_BP].base == CFI_BP)
2499 		return true;
2500 
2501 	return false;
2502 }
2503 
update_cfi_state_regs(struct instruction * insn,struct cfi_state * cfi,struct stack_op * op)2504 static int update_cfi_state_regs(struct instruction *insn,
2505 				  struct cfi_state *cfi,
2506 				  struct stack_op *op)
2507 {
2508 	struct cfi_reg *cfa = &cfi->cfa;
2509 
2510 	if (cfa->base != CFI_SP && cfa->base != CFI_SP_INDIRECT)
2511 		return 0;
2512 
2513 	/* push */
2514 	if (op->dest.type == OP_DEST_PUSH || op->dest.type == OP_DEST_PUSHF)
2515 		cfa->offset += 8;
2516 
2517 	/* pop */
2518 	if (op->src.type == OP_SRC_POP || op->src.type == OP_SRC_POPF)
2519 		cfa->offset -= 8;
2520 
2521 	/* add immediate to sp */
2522 	if (op->dest.type == OP_DEST_REG && op->src.type == OP_SRC_ADD &&
2523 	    op->dest.reg == CFI_SP && op->src.reg == CFI_SP)
2524 		cfa->offset -= op->src.offset;
2525 
2526 	return 0;
2527 }
2528 
save_reg(struct cfi_state * cfi,unsigned char reg,int base,int offset)2529 static void save_reg(struct cfi_state *cfi, unsigned char reg, int base, int offset)
2530 {
2531 	if (arch_callee_saved_reg(reg) &&
2532 	    cfi->regs[reg].base == CFI_UNDEFINED) {
2533 		cfi->regs[reg].base = base;
2534 		cfi->regs[reg].offset = offset;
2535 	}
2536 }
2537 
restore_reg(struct cfi_state * cfi,unsigned char reg)2538 static void restore_reg(struct cfi_state *cfi, unsigned char reg)
2539 {
2540 	cfi->regs[reg].base = initial_func_cfi.regs[reg].base;
2541 	cfi->regs[reg].offset = initial_func_cfi.regs[reg].offset;
2542 }
2543 
2544 /*
2545  * A note about DRAP stack alignment:
2546  *
2547  * GCC has the concept of a DRAP register, which is used to help keep track of
2548  * the stack pointer when aligning the stack.  r10 or r13 is used as the DRAP
2549  * register.  The typical DRAP pattern is:
2550  *
2551  *   4c 8d 54 24 08		lea    0x8(%rsp),%r10
2552  *   48 83 e4 c0		and    $0xffffffffffffffc0,%rsp
2553  *   41 ff 72 f8		pushq  -0x8(%r10)
2554  *   55				push   %rbp
2555  *   48 89 e5			mov    %rsp,%rbp
2556  *				(more pushes)
2557  *   41 52			push   %r10
2558  *				...
2559  *   41 5a			pop    %r10
2560  *				(more pops)
2561  *   5d				pop    %rbp
2562  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2563  *   c3				retq
2564  *
2565  * There are some variations in the epilogues, like:
2566  *
2567  *   5b				pop    %rbx
2568  *   41 5a			pop    %r10
2569  *   41 5c			pop    %r12
2570  *   41 5d			pop    %r13
2571  *   41 5e			pop    %r14
2572  *   c9				leaveq
2573  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2574  *   c3				retq
2575  *
2576  * and:
2577  *
2578  *   4c 8b 55 e8		mov    -0x18(%rbp),%r10
2579  *   48 8b 5d e0		mov    -0x20(%rbp),%rbx
2580  *   4c 8b 65 f0		mov    -0x10(%rbp),%r12
2581  *   4c 8b 6d f8		mov    -0x8(%rbp),%r13
2582  *   c9				leaveq
2583  *   49 8d 62 f8		lea    -0x8(%r10),%rsp
2584  *   c3				retq
2585  *
2586  * Sometimes r13 is used as the DRAP register, in which case it's saved and
2587  * restored beforehand:
2588  *
2589  *   41 55			push   %r13
2590  *   4c 8d 6c 24 10		lea    0x10(%rsp),%r13
2591  *   48 83 e4 f0		and    $0xfffffffffffffff0,%rsp
2592  *				...
2593  *   49 8d 65 f0		lea    -0x10(%r13),%rsp
2594  *   41 5d			pop    %r13
2595  *   c3				retq
2596  */
update_cfi_state(struct instruction * insn,struct instruction * next_insn,struct cfi_state * cfi,struct stack_op * op)2597 static int update_cfi_state(struct instruction *insn,
2598 			    struct instruction *next_insn,
2599 			    struct cfi_state *cfi, struct stack_op *op)
2600 {
2601 	struct cfi_reg *cfa = &cfi->cfa;
2602 	struct cfi_reg *regs = cfi->regs;
2603 
2604 	/* stack operations don't make sense with an undefined CFA */
2605 	if (cfa->base == CFI_UNDEFINED) {
2606 		if (insn->func) {
2607 			WARN_FUNC("undefined stack state", insn->sec, insn->offset);
2608 			return -1;
2609 		}
2610 		return 0;
2611 	}
2612 
2613 	if (cfi->type == UNWIND_HINT_TYPE_REGS ||
2614 	    cfi->type == UNWIND_HINT_TYPE_REGS_PARTIAL)
2615 		return update_cfi_state_regs(insn, cfi, op);
2616 
2617 	switch (op->dest.type) {
2618 
2619 	case OP_DEST_REG:
2620 		switch (op->src.type) {
2621 
2622 		case OP_SRC_REG:
2623 			if (op->src.reg == CFI_SP && op->dest.reg == CFI_BP &&
2624 			    cfa->base == CFI_SP &&
2625 			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset)) {
2626 
2627 				/* mov %rsp, %rbp */
2628 				cfa->base = op->dest.reg;
2629 				cfi->bp_scratch = false;
2630 			}
2631 
2632 			else if (op->src.reg == CFI_SP &&
2633 				 op->dest.reg == CFI_BP && cfi->drap) {
2634 
2635 				/* drap: mov %rsp, %rbp */
2636 				regs[CFI_BP].base = CFI_BP;
2637 				regs[CFI_BP].offset = -cfi->stack_size;
2638 				cfi->bp_scratch = false;
2639 			}
2640 
2641 			else if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2642 
2643 				/*
2644 				 * mov %rsp, %reg
2645 				 *
2646 				 * This is needed for the rare case where GCC
2647 				 * does:
2648 				 *
2649 				 *   mov    %rsp, %rax
2650 				 *   ...
2651 				 *   mov    %rax, %rsp
2652 				 */
2653 				cfi->vals[op->dest.reg].base = CFI_CFA;
2654 				cfi->vals[op->dest.reg].offset = -cfi->stack_size;
2655 			}
2656 
2657 			else if (op->src.reg == CFI_BP && op->dest.reg == CFI_SP &&
2658 				 (cfa->base == CFI_BP || cfa->base == cfi->drap_reg)) {
2659 
2660 				/*
2661 				 * mov %rbp, %rsp
2662 				 *
2663 				 * Restore the original stack pointer (Clang).
2664 				 */
2665 				cfi->stack_size = -cfi->regs[CFI_BP].offset;
2666 			}
2667 
2668 			else if (op->dest.reg == cfa->base) {
2669 
2670 				/* mov %reg, %rsp */
2671 				if (cfa->base == CFI_SP &&
2672 				    cfi->vals[op->src.reg].base == CFI_CFA) {
2673 
2674 					/*
2675 					 * This is needed for the rare case
2676 					 * where GCC does something dumb like:
2677 					 *
2678 					 *   lea    0x8(%rsp), %rcx
2679 					 *   ...
2680 					 *   mov    %rcx, %rsp
2681 					 */
2682 					cfa->offset = -cfi->vals[op->src.reg].offset;
2683 					cfi->stack_size = cfa->offset;
2684 
2685 				} else if (cfa->base == CFI_SP &&
2686 					   cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2687 					   cfi->vals[op->src.reg].offset == cfa->offset) {
2688 
2689 					/*
2690 					 * Stack swizzle:
2691 					 *
2692 					 * 1: mov %rsp, (%[tos])
2693 					 * 2: mov %[tos], %rsp
2694 					 *    ...
2695 					 * 3: pop %rsp
2696 					 *
2697 					 * Where:
2698 					 *
2699 					 * 1 - places a pointer to the previous
2700 					 *     stack at the Top-of-Stack of the
2701 					 *     new stack.
2702 					 *
2703 					 * 2 - switches to the new stack.
2704 					 *
2705 					 * 3 - pops the Top-of-Stack to restore
2706 					 *     the original stack.
2707 					 *
2708 					 * Note: we set base to SP_INDIRECT
2709 					 * here and preserve offset. Therefore
2710 					 * when the unwinder reaches ToS it
2711 					 * will dereference SP and then add the
2712 					 * offset to find the next frame, IOW:
2713 					 * (%rsp) + offset.
2714 					 */
2715 					cfa->base = CFI_SP_INDIRECT;
2716 
2717 				} else {
2718 					cfa->base = CFI_UNDEFINED;
2719 					cfa->offset = 0;
2720 				}
2721 			}
2722 
2723 			else if (op->dest.reg == CFI_SP &&
2724 				 cfi->vals[op->src.reg].base == CFI_SP_INDIRECT &&
2725 				 cfi->vals[op->src.reg].offset == cfa->offset) {
2726 
2727 				/*
2728 				 * The same stack swizzle case 2) as above. But
2729 				 * because we can't change cfa->base, case 3)
2730 				 * will become a regular POP. Pretend we're a
2731 				 * PUSH so things don't go unbalanced.
2732 				 */
2733 				cfi->stack_size += 8;
2734 			}
2735 
2736 
2737 			break;
2738 
2739 		case OP_SRC_ADD:
2740 			if (op->dest.reg == CFI_SP && op->src.reg == CFI_SP) {
2741 
2742 				/* add imm, %rsp */
2743 				cfi->stack_size -= op->src.offset;
2744 				if (cfa->base == CFI_SP)
2745 					cfa->offset -= op->src.offset;
2746 				break;
2747 			}
2748 
2749 			if (op->dest.reg == CFI_SP && op->src.reg == CFI_BP) {
2750 
2751 				/* lea disp(%rbp), %rsp */
2752 				cfi->stack_size = -(op->src.offset + regs[CFI_BP].offset);
2753 				break;
2754 			}
2755 
2756 			if (!cfi->drap && op->src.reg == CFI_SP &&
2757 			    op->dest.reg == CFI_BP && cfa->base == CFI_SP &&
2758 			    check_reg_frame_pos(&regs[CFI_BP], -cfa->offset + op->src.offset)) {
2759 
2760 				/* lea disp(%rsp), %rbp */
2761 				cfa->base = CFI_BP;
2762 				cfa->offset -= op->src.offset;
2763 				cfi->bp_scratch = false;
2764 				break;
2765 			}
2766 
2767 			if (op->src.reg == CFI_SP && cfa->base == CFI_SP) {
2768 
2769 				/* drap: lea disp(%rsp), %drap */
2770 				cfi->drap_reg = op->dest.reg;
2771 
2772 				/*
2773 				 * lea disp(%rsp), %reg
2774 				 *
2775 				 * This is needed for the rare case where GCC
2776 				 * does something dumb like:
2777 				 *
2778 				 *   lea    0x8(%rsp), %rcx
2779 				 *   ...
2780 				 *   mov    %rcx, %rsp
2781 				 */
2782 				cfi->vals[op->dest.reg].base = CFI_CFA;
2783 				cfi->vals[op->dest.reg].offset = \
2784 					-cfi->stack_size + op->src.offset;
2785 
2786 				break;
2787 			}
2788 
2789 			if (cfi->drap && op->dest.reg == CFI_SP &&
2790 			    op->src.reg == cfi->drap_reg) {
2791 
2792 				 /* drap: lea disp(%drap), %rsp */
2793 				cfa->base = CFI_SP;
2794 				cfa->offset = cfi->stack_size = -op->src.offset;
2795 				cfi->drap_reg = CFI_UNDEFINED;
2796 				cfi->drap = false;
2797 				break;
2798 			}
2799 
2800 			if (op->dest.reg == cfi->cfa.base && !(next_insn && next_insn->hint)) {
2801 				WARN_FUNC("unsupported stack register modification",
2802 					  insn->sec, insn->offset);
2803 				return -1;
2804 			}
2805 
2806 			break;
2807 
2808 		case OP_SRC_AND:
2809 			if (op->dest.reg != CFI_SP ||
2810 			    (cfi->drap_reg != CFI_UNDEFINED && cfa->base != CFI_SP) ||
2811 			    (cfi->drap_reg == CFI_UNDEFINED && cfa->base != CFI_BP)) {
2812 				WARN_FUNC("unsupported stack pointer realignment",
2813 					  insn->sec, insn->offset);
2814 				return -1;
2815 			}
2816 
2817 			if (cfi->drap_reg != CFI_UNDEFINED) {
2818 				/* drap: and imm, %rsp */
2819 				cfa->base = cfi->drap_reg;
2820 				cfa->offset = cfi->stack_size = 0;
2821 				cfi->drap = true;
2822 			}
2823 
2824 			/*
2825 			 * Older versions of GCC (4.8ish) realign the stack
2826 			 * without DRAP, with a frame pointer.
2827 			 */
2828 
2829 			break;
2830 
2831 		case OP_SRC_POP:
2832 		case OP_SRC_POPF:
2833 			if (op->dest.reg == CFI_SP && cfa->base == CFI_SP_INDIRECT) {
2834 
2835 				/* pop %rsp; # restore from a stack swizzle */
2836 				cfa->base = CFI_SP;
2837 				break;
2838 			}
2839 
2840 			if (!cfi->drap && op->dest.reg == cfa->base) {
2841 
2842 				/* pop %rbp */
2843 				cfa->base = CFI_SP;
2844 			}
2845 
2846 			if (cfi->drap && cfa->base == CFI_BP_INDIRECT &&
2847 			    op->dest.reg == cfi->drap_reg &&
2848 			    cfi->drap_offset == -cfi->stack_size) {
2849 
2850 				/* drap: pop %drap */
2851 				cfa->base = cfi->drap_reg;
2852 				cfa->offset = 0;
2853 				cfi->drap_offset = -1;
2854 
2855 			} else if (cfi->stack_size == -regs[op->dest.reg].offset) {
2856 
2857 				/* pop %reg */
2858 				restore_reg(cfi, op->dest.reg);
2859 			}
2860 
2861 			cfi->stack_size -= 8;
2862 			if (cfa->base == CFI_SP)
2863 				cfa->offset -= 8;
2864 
2865 			break;
2866 
2867 		case OP_SRC_REG_INDIRECT:
2868 			if (!cfi->drap && op->dest.reg == cfa->base &&
2869 			    op->dest.reg == CFI_BP) {
2870 
2871 				/* mov disp(%rsp), %rbp */
2872 				cfa->base = CFI_SP;
2873 				cfa->offset = cfi->stack_size;
2874 			}
2875 
2876 			if (cfi->drap && op->src.reg == CFI_BP &&
2877 			    op->src.offset == cfi->drap_offset) {
2878 
2879 				/* drap: mov disp(%rbp), %drap */
2880 				cfa->base = cfi->drap_reg;
2881 				cfa->offset = 0;
2882 				cfi->drap_offset = -1;
2883 			}
2884 
2885 			if (cfi->drap && op->src.reg == CFI_BP &&
2886 			    op->src.offset == regs[op->dest.reg].offset) {
2887 
2888 				/* drap: mov disp(%rbp), %reg */
2889 				restore_reg(cfi, op->dest.reg);
2890 
2891 			} else if (op->src.reg == cfa->base &&
2892 			    op->src.offset == regs[op->dest.reg].offset + cfa->offset) {
2893 
2894 				/* mov disp(%rbp), %reg */
2895 				/* mov disp(%rsp), %reg */
2896 				restore_reg(cfi, op->dest.reg);
2897 
2898 			} else if (op->src.reg == CFI_SP &&
2899 				   op->src.offset == regs[op->dest.reg].offset + cfi->stack_size) {
2900 
2901 				/* mov disp(%rsp), %reg */
2902 				restore_reg(cfi, op->dest.reg);
2903 			}
2904 
2905 			break;
2906 
2907 		default:
2908 			WARN_FUNC("unknown stack-related instruction",
2909 				  insn->sec, insn->offset);
2910 			return -1;
2911 		}
2912 
2913 		break;
2914 
2915 	case OP_DEST_PUSH:
2916 	case OP_DEST_PUSHF:
2917 		cfi->stack_size += 8;
2918 		if (cfa->base == CFI_SP)
2919 			cfa->offset += 8;
2920 
2921 		if (op->src.type != OP_SRC_REG)
2922 			break;
2923 
2924 		if (cfi->drap) {
2925 			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2926 
2927 				/* drap: push %drap */
2928 				cfa->base = CFI_BP_INDIRECT;
2929 				cfa->offset = -cfi->stack_size;
2930 
2931 				/* save drap so we know when to restore it */
2932 				cfi->drap_offset = -cfi->stack_size;
2933 
2934 			} else if (op->src.reg == CFI_BP && cfa->base == cfi->drap_reg) {
2935 
2936 				/* drap: push %rbp */
2937 				cfi->stack_size = 0;
2938 
2939 			} else {
2940 
2941 				/* drap: push %reg */
2942 				save_reg(cfi, op->src.reg, CFI_BP, -cfi->stack_size);
2943 			}
2944 
2945 		} else {
2946 
2947 			/* push %reg */
2948 			save_reg(cfi, op->src.reg, CFI_CFA, -cfi->stack_size);
2949 		}
2950 
2951 		/* detect when asm code uses rbp as a scratch register */
2952 		if (opts.stackval && insn->func && op->src.reg == CFI_BP &&
2953 		    cfa->base != CFI_BP)
2954 			cfi->bp_scratch = true;
2955 		break;
2956 
2957 	case OP_DEST_REG_INDIRECT:
2958 
2959 		if (cfi->drap) {
2960 			if (op->src.reg == cfa->base && op->src.reg == cfi->drap_reg) {
2961 
2962 				/* drap: mov %drap, disp(%rbp) */
2963 				cfa->base = CFI_BP_INDIRECT;
2964 				cfa->offset = op->dest.offset;
2965 
2966 				/* save drap offset so we know when to restore it */
2967 				cfi->drap_offset = op->dest.offset;
2968 			} else {
2969 
2970 				/* drap: mov reg, disp(%rbp) */
2971 				save_reg(cfi, op->src.reg, CFI_BP, op->dest.offset);
2972 			}
2973 
2974 		} else if (op->dest.reg == cfa->base) {
2975 
2976 			/* mov reg, disp(%rbp) */
2977 			/* mov reg, disp(%rsp) */
2978 			save_reg(cfi, op->src.reg, CFI_CFA,
2979 				 op->dest.offset - cfi->cfa.offset);
2980 
2981 		} else if (op->dest.reg == CFI_SP) {
2982 
2983 			/* mov reg, disp(%rsp) */
2984 			save_reg(cfi, op->src.reg, CFI_CFA,
2985 				 op->dest.offset - cfi->stack_size);
2986 
2987 		} else if (op->src.reg == CFI_SP && op->dest.offset == 0) {
2988 
2989 			/* mov %rsp, (%reg); # setup a stack swizzle. */
2990 			cfi->vals[op->dest.reg].base = CFI_SP_INDIRECT;
2991 			cfi->vals[op->dest.reg].offset = cfa->offset;
2992 		}
2993 
2994 		break;
2995 
2996 	case OP_DEST_MEM:
2997 		if (op->src.type != OP_SRC_POP && op->src.type != OP_SRC_POPF) {
2998 			WARN_FUNC("unknown stack-related memory operation",
2999 				  insn->sec, insn->offset);
3000 			return -1;
3001 		}
3002 
3003 		/* pop mem */
3004 		cfi->stack_size -= 8;
3005 		if (cfa->base == CFI_SP)
3006 			cfa->offset -= 8;
3007 
3008 		break;
3009 
3010 	default:
3011 		WARN_FUNC("unknown stack-related instruction",
3012 			  insn->sec, insn->offset);
3013 		return -1;
3014 	}
3015 
3016 	return 0;
3017 }
3018 
3019 /*
3020  * The stack layouts of alternatives instructions can sometimes diverge when
3021  * they have stack modifications.  That's fine as long as the potential stack
3022  * layouts don't conflict at any given potential instruction boundary.
3023  *
3024  * Flatten the CFIs of the different alternative code streams (both original
3025  * and replacement) into a single shared CFI array which can be used to detect
3026  * conflicts and nicely feed a linear array of ORC entries to the unwinder.
3027  */
propagate_alt_cfi(struct objtool_file * file,struct instruction * insn)3028 static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn)
3029 {
3030 	struct cfi_state **alt_cfi;
3031 	int group_off;
3032 
3033 	if (!insn->alt_group)
3034 		return 0;
3035 
3036 	if (!insn->cfi) {
3037 		WARN("CFI missing");
3038 		return -1;
3039 	}
3040 
3041 	alt_cfi = insn->alt_group->cfi;
3042 	group_off = insn->offset - insn->alt_group->first_insn->offset;
3043 
3044 	if (!alt_cfi[group_off]) {
3045 		alt_cfi[group_off] = insn->cfi;
3046 	} else {
3047 		if (cficmp(alt_cfi[group_off], insn->cfi)) {
3048 			WARN_FUNC("stack layout conflict in alternatives",
3049 				  insn->sec, insn->offset);
3050 			return -1;
3051 		}
3052 	}
3053 
3054 	return 0;
3055 }
3056 
handle_insn_ops(struct instruction * insn,struct instruction * next_insn,struct insn_state * state)3057 static int handle_insn_ops(struct instruction *insn,
3058 			   struct instruction *next_insn,
3059 			   struct insn_state *state)
3060 {
3061 	struct stack_op *op;
3062 
3063 	list_for_each_entry(op, &insn->stack_ops, list) {
3064 
3065 		if (update_cfi_state(insn, next_insn, &state->cfi, op))
3066 			return 1;
3067 
3068 		if (!insn->alt_group)
3069 			continue;
3070 
3071 		if (op->dest.type == OP_DEST_PUSHF) {
3072 			if (!state->uaccess_stack) {
3073 				state->uaccess_stack = 1;
3074 			} else if (state->uaccess_stack >> 31) {
3075 				WARN_FUNC("PUSHF stack exhausted",
3076 					  insn->sec, insn->offset);
3077 				return 1;
3078 			}
3079 			state->uaccess_stack <<= 1;
3080 			state->uaccess_stack  |= state->uaccess;
3081 		}
3082 
3083 		if (op->src.type == OP_SRC_POPF) {
3084 			if (state->uaccess_stack) {
3085 				state->uaccess = state->uaccess_stack & 1;
3086 				state->uaccess_stack >>= 1;
3087 				if (state->uaccess_stack == 1)
3088 					state->uaccess_stack = 0;
3089 			}
3090 		}
3091 	}
3092 
3093 	return 0;
3094 }
3095 
insn_cfi_match(struct instruction * insn,struct cfi_state * cfi2)3096 static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2)
3097 {
3098 	struct cfi_state *cfi1 = insn->cfi;
3099 	int i;
3100 
3101 	if (!cfi1) {
3102 		WARN("CFI missing");
3103 		return false;
3104 	}
3105 
3106 	if (memcmp(&cfi1->cfa, &cfi2->cfa, sizeof(cfi1->cfa))) {
3107 
3108 		WARN_FUNC("stack state mismatch: cfa1=%d%+d cfa2=%d%+d",
3109 			  insn->sec, insn->offset,
3110 			  cfi1->cfa.base, cfi1->cfa.offset,
3111 			  cfi2->cfa.base, cfi2->cfa.offset);
3112 
3113 	} else if (memcmp(&cfi1->regs, &cfi2->regs, sizeof(cfi1->regs))) {
3114 		for (i = 0; i < CFI_NUM_REGS; i++) {
3115 			if (!memcmp(&cfi1->regs[i], &cfi2->regs[i],
3116 				    sizeof(struct cfi_reg)))
3117 				continue;
3118 
3119 			WARN_FUNC("stack state mismatch: reg1[%d]=%d%+d reg2[%d]=%d%+d",
3120 				  insn->sec, insn->offset,
3121 				  i, cfi1->regs[i].base, cfi1->regs[i].offset,
3122 				  i, cfi2->regs[i].base, cfi2->regs[i].offset);
3123 			break;
3124 		}
3125 
3126 	} else if (cfi1->type != cfi2->type) {
3127 
3128 		WARN_FUNC("stack state mismatch: type1=%d type2=%d",
3129 			  insn->sec, insn->offset, cfi1->type, cfi2->type);
3130 
3131 	} else if (cfi1->drap != cfi2->drap ||
3132 		   (cfi1->drap && cfi1->drap_reg != cfi2->drap_reg) ||
3133 		   (cfi1->drap && cfi1->drap_offset != cfi2->drap_offset)) {
3134 
3135 		WARN_FUNC("stack state mismatch: drap1=%d(%d,%d) drap2=%d(%d,%d)",
3136 			  insn->sec, insn->offset,
3137 			  cfi1->drap, cfi1->drap_reg, cfi1->drap_offset,
3138 			  cfi2->drap, cfi2->drap_reg, cfi2->drap_offset);
3139 
3140 	} else
3141 		return true;
3142 
3143 	return false;
3144 }
3145 
func_uaccess_safe(struct symbol * func)3146 static inline bool func_uaccess_safe(struct symbol *func)
3147 {
3148 	if (func)
3149 		return func->uaccess_safe;
3150 
3151 	return false;
3152 }
3153 
call_dest_name(struct instruction * insn)3154 static inline const char *call_dest_name(struct instruction *insn)
3155 {
3156 	static char pvname[19];
3157 	struct reloc *rel;
3158 	int idx;
3159 
3160 	if (insn->call_dest)
3161 		return insn->call_dest->name;
3162 
3163 	rel = insn_reloc(NULL, insn);
3164 	if (rel && !strcmp(rel->sym->name, "pv_ops")) {
3165 		idx = (rel->addend / sizeof(void *));
3166 		snprintf(pvname, sizeof(pvname), "pv_ops[%d]", idx);
3167 		return pvname;
3168 	}
3169 
3170 	return "{dynamic}";
3171 }
3172 
pv_call_dest(struct objtool_file * file,struct instruction * insn)3173 static bool pv_call_dest(struct objtool_file *file, struct instruction *insn)
3174 {
3175 	struct symbol *target;
3176 	struct reloc *rel;
3177 	int idx;
3178 
3179 	rel = insn_reloc(file, insn);
3180 	if (!rel || strcmp(rel->sym->name, "pv_ops"))
3181 		return false;
3182 
3183 	idx = (arch_dest_reloc_offset(rel->addend) / sizeof(void *));
3184 
3185 	if (file->pv_ops[idx].clean)
3186 		return true;
3187 
3188 	file->pv_ops[idx].clean = true;
3189 
3190 	list_for_each_entry(target, &file->pv_ops[idx].targets, pv_target) {
3191 		if (!target->sec->noinstr) {
3192 			WARN("pv_ops[%d]: %s", idx, target->name);
3193 			file->pv_ops[idx].clean = false;
3194 		}
3195 	}
3196 
3197 	return file->pv_ops[idx].clean;
3198 }
3199 
noinstr_call_dest(struct objtool_file * file,struct instruction * insn,struct symbol * func)3200 static inline bool noinstr_call_dest(struct objtool_file *file,
3201 				     struct instruction *insn,
3202 				     struct symbol *func)
3203 {
3204 	/*
3205 	 * We can't deal with indirect function calls at present;
3206 	 * assume they're instrumented.
3207 	 */
3208 	if (!func) {
3209 		if (file->pv_ops)
3210 			return pv_call_dest(file, insn);
3211 
3212 		return false;
3213 	}
3214 
3215 	/*
3216 	 * If the symbol is from a noinstr section; we good.
3217 	 */
3218 	if (func->sec->noinstr)
3219 		return true;
3220 
3221 	/*
3222 	 * The __ubsan_handle_*() calls are like WARN(), they only happen when
3223 	 * something 'BAD' happened. At the risk of taking the machine down,
3224 	 * let them proceed to get the message out.
3225 	 */
3226 	if (!strncmp(func->name, "__ubsan_handle_", 15))
3227 		return true;
3228 
3229 	return false;
3230 }
3231 
validate_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3232 static int validate_call(struct objtool_file *file,
3233 			 struct instruction *insn,
3234 			 struct insn_state *state)
3235 {
3236 	if (state->noinstr && state->instr <= 0 &&
3237 	    !noinstr_call_dest(file, insn, insn->call_dest)) {
3238 		WARN_FUNC("call to %s() leaves .noinstr.text section",
3239 				insn->sec, insn->offset, call_dest_name(insn));
3240 		return 1;
3241 	}
3242 
3243 	if (state->uaccess && !func_uaccess_safe(insn->call_dest)) {
3244 		WARN_FUNC("call to %s() with UACCESS enabled",
3245 				insn->sec, insn->offset, call_dest_name(insn));
3246 		return 1;
3247 	}
3248 
3249 	if (state->df) {
3250 		WARN_FUNC("call to %s() with DF set",
3251 				insn->sec, insn->offset, call_dest_name(insn));
3252 		return 1;
3253 	}
3254 
3255 	return 0;
3256 }
3257 
validate_sibling_call(struct objtool_file * file,struct instruction * insn,struct insn_state * state)3258 static int validate_sibling_call(struct objtool_file *file,
3259 				 struct instruction *insn,
3260 				 struct insn_state *state)
3261 {
3262 	if (has_modified_stack_frame(insn, state)) {
3263 		WARN_FUNC("sibling call from callable instruction with modified stack frame",
3264 				insn->sec, insn->offset);
3265 		return 1;
3266 	}
3267 
3268 	return validate_call(file, insn, state);
3269 }
3270 
validate_return(struct symbol * func,struct instruction * insn,struct insn_state * state)3271 static int validate_return(struct symbol *func, struct instruction *insn, struct insn_state *state)
3272 {
3273 	if (state->noinstr && state->instr > 0) {
3274 		WARN_FUNC("return with instrumentation enabled",
3275 			  insn->sec, insn->offset);
3276 		return 1;
3277 	}
3278 
3279 	if (state->uaccess && !func_uaccess_safe(func)) {
3280 		WARN_FUNC("return with UACCESS enabled",
3281 			  insn->sec, insn->offset);
3282 		return 1;
3283 	}
3284 
3285 	if (!state->uaccess && func_uaccess_safe(func)) {
3286 		WARN_FUNC("return with UACCESS disabled from a UACCESS-safe function",
3287 			  insn->sec, insn->offset);
3288 		return 1;
3289 	}
3290 
3291 	if (state->df) {
3292 		WARN_FUNC("return with DF set",
3293 			  insn->sec, insn->offset);
3294 		return 1;
3295 	}
3296 
3297 	if (func && has_modified_stack_frame(insn, state)) {
3298 		WARN_FUNC("return with modified stack frame",
3299 			  insn->sec, insn->offset);
3300 		return 1;
3301 	}
3302 
3303 	if (state->cfi.bp_scratch) {
3304 		WARN_FUNC("BP used as a scratch register",
3305 			  insn->sec, insn->offset);
3306 		return 1;
3307 	}
3308 
3309 	return 0;
3310 }
3311 
next_insn_to_validate(struct objtool_file * file,struct instruction * insn)3312 static struct instruction *next_insn_to_validate(struct objtool_file *file,
3313 						 struct instruction *insn)
3314 {
3315 	struct alt_group *alt_group = insn->alt_group;
3316 
3317 	/*
3318 	 * Simulate the fact that alternatives are patched in-place.  When the
3319 	 * end of a replacement alt_group is reached, redirect objtool flow to
3320 	 * the end of the original alt_group.
3321 	 */
3322 	if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3323 		return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3324 
3325 	return next_insn_same_sec(file, insn);
3326 }
3327 
3328 /*
3329  * Follow the branch starting at the given instruction, and recursively follow
3330  * any other branches (jumps).  Meanwhile, track the frame pointer state at
3331  * each instruction and validate all the rules described in
3332  * tools/objtool/Documentation/objtool.txt.
3333  */
validate_branch(struct objtool_file * file,struct symbol * func,struct instruction * insn,struct insn_state state)3334 static int validate_branch(struct objtool_file *file, struct symbol *func,
3335 			   struct instruction *insn, struct insn_state state)
3336 {
3337 	struct alternative *alt;
3338 	struct instruction *next_insn, *prev_insn = NULL;
3339 	struct section *sec;
3340 	u8 visited;
3341 	int ret;
3342 
3343 	sec = insn->sec;
3344 
3345 	while (1) {
3346 		next_insn = next_insn_to_validate(file, insn);
3347 
3348 		if (func && insn->func && func != insn->func->pfunc) {
3349 			/* Ignore KCFI type preambles, which always fall through */
3350 			if (!strncmp(func->name, "__cfi_", 6))
3351 				return 0;
3352 
3353 			WARN("%s() falls through to next function %s()",
3354 			     func->name, insn->func->name);
3355 			return 1;
3356 		}
3357 
3358 		if (func && insn->ignore) {
3359 			WARN_FUNC("BUG: why am I validating an ignored function?",
3360 				  sec, insn->offset);
3361 			return 1;
3362 		}
3363 
3364 		visited = VISITED_BRANCH << state.uaccess;
3365 		if (insn->visited & VISITED_BRANCH_MASK) {
3366 			if (!insn->hint && !insn_cfi_match(insn, &state.cfi))
3367 				return 1;
3368 
3369 			if (insn->visited & visited)
3370 				return 0;
3371 		} else {
3372 			nr_insns_visited++;
3373 		}
3374 
3375 		if (state.noinstr)
3376 			state.instr += insn->instr;
3377 
3378 		if (insn->hint) {
3379 			if (insn->restore) {
3380 				struct instruction *save_insn, *i;
3381 
3382 				i = insn;
3383 				save_insn = NULL;
3384 
3385 				sym_for_each_insn_continue_reverse(file, func, i) {
3386 					if (i->save) {
3387 						save_insn = i;
3388 						break;
3389 					}
3390 				}
3391 
3392 				if (!save_insn) {
3393 					WARN_FUNC("no corresponding CFI save for CFI restore",
3394 						  sec, insn->offset);
3395 					return 1;
3396 				}
3397 
3398 				if (!save_insn->visited) {
3399 					WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo",
3400 						  sec, insn->offset);
3401 					return 1;
3402 				}
3403 
3404 				insn->cfi = save_insn->cfi;
3405 				nr_cfi_reused++;
3406 			}
3407 
3408 			state.cfi = *insn->cfi;
3409 		} else {
3410 			/* XXX track if we actually changed state.cfi */
3411 
3412 			if (prev_insn && !cficmp(prev_insn->cfi, &state.cfi)) {
3413 				insn->cfi = prev_insn->cfi;
3414 				nr_cfi_reused++;
3415 			} else {
3416 				insn->cfi = cfi_hash_find_or_add(&state.cfi);
3417 			}
3418 		}
3419 
3420 		insn->visited |= visited;
3421 
3422 		if (propagate_alt_cfi(file, insn))
3423 			return 1;
3424 
3425 		if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3426 			bool skip_orig = false;
3427 
3428 			list_for_each_entry(alt, &insn->alts, list) {
3429 				if (alt->skip_orig)
3430 					skip_orig = true;
3431 
3432 				ret = validate_branch(file, func, alt->insn, state);
3433 				if (ret) {
3434 					if (opts.backtrace)
3435 						BT_FUNC("(alt)", insn);
3436 					return ret;
3437 				}
3438 			}
3439 
3440 			if (skip_orig)
3441 				return 0;
3442 		}
3443 
3444 		if (handle_insn_ops(insn, next_insn, &state))
3445 			return 1;
3446 
3447 		switch (insn->type) {
3448 
3449 		case INSN_RETURN:
3450 			return validate_return(func, insn, &state);
3451 
3452 		case INSN_CALL:
3453 		case INSN_CALL_DYNAMIC:
3454 			ret = validate_call(file, insn, &state);
3455 			if (ret)
3456 				return ret;
3457 
3458 			if (opts.stackval && func && !is_fentry_call(insn) &&
3459 			    !has_valid_stack_frame(&state)) {
3460 				WARN_FUNC("call without frame pointer save/setup",
3461 					  sec, insn->offset);
3462 				return 1;
3463 			}
3464 
3465 			if (insn->dead_end)
3466 				return 0;
3467 
3468 			break;
3469 
3470 		case INSN_JUMP_CONDITIONAL:
3471 		case INSN_JUMP_UNCONDITIONAL:
3472 			if (is_sibling_call(insn)) {
3473 				ret = validate_sibling_call(file, insn, &state);
3474 				if (ret)
3475 					return ret;
3476 
3477 			} else if (insn->jump_dest) {
3478 				ret = validate_branch(file, func,
3479 						      insn->jump_dest, state);
3480 				if (ret) {
3481 					if (opts.backtrace)
3482 						BT_FUNC("(branch)", insn);
3483 					return ret;
3484 				}
3485 			}
3486 
3487 			if (insn->type == INSN_JUMP_UNCONDITIONAL)
3488 				return 0;
3489 
3490 			break;
3491 
3492 		case INSN_JUMP_DYNAMIC:
3493 		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3494 			if (is_sibling_call(insn)) {
3495 				ret = validate_sibling_call(file, insn, &state);
3496 				if (ret)
3497 					return ret;
3498 			}
3499 
3500 			if (insn->type == INSN_JUMP_DYNAMIC)
3501 				return 0;
3502 
3503 			break;
3504 
3505 		case INSN_CONTEXT_SWITCH:
3506 			if (func && (!next_insn || !next_insn->hint)) {
3507 				WARN_FUNC("unsupported instruction in callable function",
3508 					  sec, insn->offset);
3509 				return 1;
3510 			}
3511 			return 0;
3512 
3513 		case INSN_STAC:
3514 			if (state.uaccess) {
3515 				WARN_FUNC("recursive UACCESS enable", sec, insn->offset);
3516 				return 1;
3517 			}
3518 
3519 			state.uaccess = true;
3520 			break;
3521 
3522 		case INSN_CLAC:
3523 			if (!state.uaccess && func) {
3524 				WARN_FUNC("redundant UACCESS disable", sec, insn->offset);
3525 				return 1;
3526 			}
3527 
3528 			if (func_uaccess_safe(func) && !state.uaccess_stack) {
3529 				WARN_FUNC("UACCESS-safe disables UACCESS", sec, insn->offset);
3530 				return 1;
3531 			}
3532 
3533 			state.uaccess = false;
3534 			break;
3535 
3536 		case INSN_STD:
3537 			if (state.df) {
3538 				WARN_FUNC("recursive STD", sec, insn->offset);
3539 				return 1;
3540 			}
3541 
3542 			state.df = true;
3543 			break;
3544 
3545 		case INSN_CLD:
3546 			if (!state.df && func) {
3547 				WARN_FUNC("redundant CLD", sec, insn->offset);
3548 				return 1;
3549 			}
3550 
3551 			state.df = false;
3552 			break;
3553 
3554 		default:
3555 			break;
3556 		}
3557 
3558 		if (insn->dead_end)
3559 			return 0;
3560 
3561 		if (!next_insn) {
3562 			if (state.cfi.cfa.base == CFI_UNDEFINED)
3563 				return 0;
3564 			WARN("%s: unexpected end of section", sec->name);
3565 			return 1;
3566 		}
3567 
3568 		prev_insn = insn;
3569 		insn = next_insn;
3570 	}
3571 
3572 	return 0;
3573 }
3574 
validate_unwind_hints(struct objtool_file * file,struct section * sec)3575 static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
3576 {
3577 	struct instruction *insn;
3578 	struct insn_state state;
3579 	int ret, warnings = 0;
3580 
3581 	if (!file->hints)
3582 		return 0;
3583 
3584 	init_insn_state(file, &state, sec);
3585 
3586 	if (sec) {
3587 		insn = find_insn(file, sec, 0);
3588 		if (!insn)
3589 			return 0;
3590 	} else {
3591 		insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3592 	}
3593 
3594 	while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3595 		if (insn->hint && !insn->visited && !insn->ignore) {
3596 			ret = validate_branch(file, insn->func, insn, state);
3597 			if (ret && opts.backtrace)
3598 				BT_FUNC("<=== (hint)", insn);
3599 			warnings += ret;
3600 		}
3601 
3602 		insn = list_next_entry(insn, list);
3603 	}
3604 
3605 	return warnings;
3606 }
3607 
3608 /*
3609  * Validate rethunk entry constraint: must untrain RET before the first RET.
3610  *
3611  * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes
3612  * before an actual RET instruction.
3613  */
validate_entry(struct objtool_file * file,struct instruction * insn)3614 static int validate_entry(struct objtool_file *file, struct instruction *insn)
3615 {
3616 	struct instruction *next, *dest;
3617 	int ret, warnings = 0;
3618 
3619 	for (;;) {
3620 		next = next_insn_to_validate(file, insn);
3621 
3622 		if (insn->visited & VISITED_ENTRY)
3623 			return 0;
3624 
3625 		insn->visited |= VISITED_ENTRY;
3626 
3627 		if (!insn->ignore_alts && !list_empty(&insn->alts)) {
3628 			struct alternative *alt;
3629 			bool skip_orig = false;
3630 
3631 			list_for_each_entry(alt, &insn->alts, list) {
3632 				if (alt->skip_orig)
3633 					skip_orig = true;
3634 
3635 				ret = validate_entry(file, alt->insn);
3636 				if (ret) {
3637 				        if (opts.backtrace)
3638 						BT_FUNC("(alt)", insn);
3639 					return ret;
3640 				}
3641 			}
3642 
3643 			if (skip_orig)
3644 				return 0;
3645 		}
3646 
3647 		switch (insn->type) {
3648 
3649 		case INSN_CALL_DYNAMIC:
3650 		case INSN_JUMP_DYNAMIC:
3651 		case INSN_JUMP_DYNAMIC_CONDITIONAL:
3652 			WARN_FUNC("early indirect call", insn->sec, insn->offset);
3653 			return 1;
3654 
3655 		case INSN_JUMP_UNCONDITIONAL:
3656 		case INSN_JUMP_CONDITIONAL:
3657 			if (!is_sibling_call(insn)) {
3658 				if (!insn->jump_dest) {
3659 					WARN_FUNC("unresolved jump target after linking?!?",
3660 						  insn->sec, insn->offset);
3661 					return -1;
3662 				}
3663 				ret = validate_entry(file, insn->jump_dest);
3664 				if (ret) {
3665 					if (opts.backtrace) {
3666 						BT_FUNC("(branch%s)", insn,
3667 							insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : "");
3668 					}
3669 					return ret;
3670 				}
3671 
3672 				if (insn->type == INSN_JUMP_UNCONDITIONAL)
3673 					return 0;
3674 
3675 				break;
3676 			}
3677 
3678 			/* fallthrough */
3679 		case INSN_CALL:
3680 			dest = find_insn(file, insn->call_dest->sec,
3681 					 insn->call_dest->offset);
3682 			if (!dest) {
3683 				WARN("Unresolved function after linking!?: %s",
3684 				     insn->call_dest->name);
3685 				return -1;
3686 			}
3687 
3688 			ret = validate_entry(file, dest);
3689 			if (ret) {
3690 				if (opts.backtrace)
3691 					BT_FUNC("(call)", insn);
3692 				return ret;
3693 			}
3694 			/*
3695 			 * If a call returns without error, it must have seen UNTRAIN_RET.
3696 			 * Therefore any non-error return is a success.
3697 			 */
3698 			return 0;
3699 
3700 		case INSN_RETURN:
3701 			WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset);
3702 			return 1;
3703 
3704 		case INSN_NOP:
3705 			if (insn->retpoline_safe)
3706 				return 0;
3707 			break;
3708 
3709 		default:
3710 			break;
3711 		}
3712 
3713 		if (!next) {
3714 			WARN_FUNC("teh end!", insn->sec, insn->offset);
3715 			return -1;
3716 		}
3717 		insn = next;
3718 	}
3719 
3720 	return warnings;
3721 }
3722 
3723 /*
3724  * Validate that all branches starting at 'insn->entry' encounter UNRET_END
3725  * before RET.
3726  */
validate_unret(struct objtool_file * file)3727 static int validate_unret(struct objtool_file *file)
3728 {
3729 	struct instruction *insn;
3730 	int ret, warnings = 0;
3731 
3732 	for_each_insn(file, insn) {
3733 		if (!insn->entry)
3734 			continue;
3735 
3736 		ret = validate_entry(file, insn);
3737 		if (ret < 0) {
3738 			WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset);
3739 			return ret;
3740 		}
3741 		warnings += ret;
3742 	}
3743 
3744 	return warnings;
3745 }
3746 
validate_retpoline(struct objtool_file * file)3747 static int validate_retpoline(struct objtool_file *file)
3748 {
3749 	struct instruction *insn;
3750 	int warnings = 0;
3751 
3752 	for_each_insn(file, insn) {
3753 		if (insn->type != INSN_JUMP_DYNAMIC &&
3754 		    insn->type != INSN_CALL_DYNAMIC &&
3755 		    insn->type != INSN_RETURN)
3756 			continue;
3757 
3758 		if (insn->retpoline_safe)
3759 			continue;
3760 
3761 		/*
3762 		 * .init.text code is ran before userspace and thus doesn't
3763 		 * strictly need retpolines, except for modules which are
3764 		 * loaded late, they very much do need retpoline in their
3765 		 * .init.text
3766 		 */
3767 		if (!strcmp(insn->sec->name, ".init.text") && !opts.module)
3768 			continue;
3769 
3770 		if (insn->type == INSN_RETURN) {
3771 			if (opts.rethunk) {
3772 				WARN_FUNC("'naked' return found in RETHUNK build",
3773 					  insn->sec, insn->offset);
3774 			} else
3775 				continue;
3776 		} else {
3777 			WARN_FUNC("indirect %s found in RETPOLINE build",
3778 				  insn->sec, insn->offset,
3779 				  insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call");
3780 		}
3781 
3782 		warnings++;
3783 	}
3784 
3785 	return warnings;
3786 }
3787 
is_kasan_insn(struct instruction * insn)3788 static bool is_kasan_insn(struct instruction *insn)
3789 {
3790 	return (insn->type == INSN_CALL &&
3791 		!strcmp(insn->call_dest->name, "__asan_handle_no_return"));
3792 }
3793 
is_ubsan_insn(struct instruction * insn)3794 static bool is_ubsan_insn(struct instruction *insn)
3795 {
3796 	return (insn->type == INSN_CALL &&
3797 		!strcmp(insn->call_dest->name,
3798 			"__ubsan_handle_builtin_unreachable"));
3799 }
3800 
ignore_unreachable_insn(struct objtool_file * file,struct instruction * insn)3801 static bool ignore_unreachable_insn(struct objtool_file *file, struct instruction *insn)
3802 {
3803 	int i;
3804 	struct instruction *prev_insn;
3805 
3806 	if (insn->ignore || insn->type == INSN_NOP || insn->type == INSN_TRAP)
3807 		return true;
3808 
3809 	/*
3810 	 * Ignore alternative replacement instructions.  This can happen
3811 	 * when a whitelisted function uses one of the ALTERNATIVE macros.
3812 	 */
3813 	if (!strcmp(insn->sec->name, ".altinstr_replacement") ||
3814 	    !strcmp(insn->sec->name, ".altinstr_aux"))
3815 		return true;
3816 
3817 	/*
3818 	 * Whole archive runs might encounter dead code from weak symbols.
3819 	 * This is where the linker will have dropped the weak symbol in
3820 	 * favour of a regular symbol, but leaves the code in place.
3821 	 *
3822 	 * In this case we'll find a piece of code (whole function) that is not
3823 	 * covered by a !section symbol. Ignore them.
3824 	 */
3825 	if (opts.link && !insn->func) {
3826 		int size = find_symbol_hole_containing(insn->sec, insn->offset);
3827 		unsigned long end = insn->offset + size;
3828 
3829 		if (!size) /* not a hole */
3830 			return false;
3831 
3832 		if (size < 0) /* hole until the end */
3833 			return true;
3834 
3835 		sec_for_each_insn_continue(file, insn) {
3836 			/*
3837 			 * If we reach a visited instruction at or before the
3838 			 * end of the hole, ignore the unreachable.
3839 			 */
3840 			if (insn->visited)
3841 				return true;
3842 
3843 			if (insn->offset >= end)
3844 				break;
3845 
3846 			/*
3847 			 * If this hole jumps to a .cold function, mark it ignore too.
3848 			 */
3849 			if (insn->jump_dest && insn->jump_dest->func &&
3850 			    strstr(insn->jump_dest->func->name, ".cold")) {
3851 				struct instruction *dest = insn->jump_dest;
3852 				func_for_each_insn(file, dest->func, dest)
3853 					dest->ignore = true;
3854 			}
3855 		}
3856 
3857 		return false;
3858 	}
3859 
3860 	if (!insn->func)
3861 		return false;
3862 
3863 	if (insn->func->static_call_tramp)
3864 		return true;
3865 
3866 	/*
3867 	 * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
3868 	 * __builtin_unreachable().  The BUG() macro has an unreachable() after
3869 	 * the UD2, which causes GCC's undefined trap logic to emit another UD2
3870 	 * (or occasionally a JMP to UD2).
3871 	 *
3872 	 * It may also insert a UD2 after calling a __noreturn function.
3873 	 */
3874 	prev_insn = list_prev_entry(insn, list);
3875 	if ((prev_insn->dead_end || dead_end_function(file, prev_insn->call_dest)) &&
3876 	    (insn->type == INSN_BUG ||
3877 	     (insn->type == INSN_JUMP_UNCONDITIONAL &&
3878 	      insn->jump_dest && insn->jump_dest->type == INSN_BUG)))
3879 		return true;
3880 
3881 	/*
3882 	 * Check if this (or a subsequent) instruction is related to
3883 	 * CONFIG_UBSAN or CONFIG_KASAN.
3884 	 *
3885 	 * End the search at 5 instructions to avoid going into the weeds.
3886 	 */
3887 	for (i = 0; i < 5; i++) {
3888 
3889 		if (is_kasan_insn(insn) || is_ubsan_insn(insn))
3890 			return true;
3891 
3892 		if (insn->type == INSN_JUMP_UNCONDITIONAL) {
3893 			if (insn->jump_dest &&
3894 			    insn->jump_dest->func == insn->func) {
3895 				insn = insn->jump_dest;
3896 				continue;
3897 			}
3898 
3899 			break;
3900 		}
3901 
3902 		if (insn->offset + insn->len >= insn->func->offset + insn->func->len)
3903 			break;
3904 
3905 		insn = list_next_entry(insn, list);
3906 	}
3907 
3908 	return false;
3909 }
3910 
validate_symbol(struct objtool_file * file,struct section * sec,struct symbol * sym,struct insn_state * state)3911 static int validate_symbol(struct objtool_file *file, struct section *sec,
3912 			   struct symbol *sym, struct insn_state *state)
3913 {
3914 	struct instruction *insn;
3915 	int ret;
3916 
3917 	if (!sym->len) {
3918 		WARN("%s() is missing an ELF size annotation", sym->name);
3919 		return 1;
3920 	}
3921 
3922 	if (sym->pfunc != sym || sym->alias != sym)
3923 		return 0;
3924 
3925 	insn = find_insn(file, sec, sym->offset);
3926 	if (!insn || insn->ignore || insn->visited)
3927 		return 0;
3928 
3929 	state->uaccess = sym->uaccess_safe;
3930 
3931 	ret = validate_branch(file, insn->func, insn, *state);
3932 	if (ret && opts.backtrace)
3933 		BT_FUNC("<=== (sym)", insn);
3934 	return ret;
3935 }
3936 
validate_section(struct objtool_file * file,struct section * sec)3937 static int validate_section(struct objtool_file *file, struct section *sec)
3938 {
3939 	struct insn_state state;
3940 	struct symbol *func;
3941 	int warnings = 0;
3942 
3943 	list_for_each_entry(func, &sec->symbol_list, list) {
3944 		if (func->type != STT_FUNC)
3945 			continue;
3946 
3947 		init_insn_state(file, &state, sec);
3948 		set_func_state(&state.cfi);
3949 
3950 		warnings += validate_symbol(file, sec, func, &state);
3951 	}
3952 
3953 	return warnings;
3954 }
3955 
validate_noinstr_sections(struct objtool_file * file)3956 static int validate_noinstr_sections(struct objtool_file *file)
3957 {
3958 	struct section *sec;
3959 	int warnings = 0;
3960 
3961 	sec = find_section_by_name(file->elf, ".noinstr.text");
3962 	if (sec) {
3963 		warnings += validate_section(file, sec);
3964 		warnings += validate_unwind_hints(file, sec);
3965 	}
3966 
3967 	sec = find_section_by_name(file->elf, ".entry.text");
3968 	if (sec) {
3969 		warnings += validate_section(file, sec);
3970 		warnings += validate_unwind_hints(file, sec);
3971 	}
3972 
3973 	return warnings;
3974 }
3975 
validate_functions(struct objtool_file * file)3976 static int validate_functions(struct objtool_file *file)
3977 {
3978 	struct section *sec;
3979 	int warnings = 0;
3980 
3981 	for_each_sec(file, sec) {
3982 		if (!(sec->sh.sh_flags & SHF_EXECINSTR))
3983 			continue;
3984 
3985 		warnings += validate_section(file, sec);
3986 	}
3987 
3988 	return warnings;
3989 }
3990 
mark_endbr_used(struct instruction * insn)3991 static void mark_endbr_used(struct instruction *insn)
3992 {
3993 	if (!list_empty(&insn->call_node))
3994 		list_del_init(&insn->call_node);
3995 }
3996 
validate_ibt_insn(struct objtool_file * file,struct instruction * insn)3997 static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn)
3998 {
3999 	struct instruction *dest;
4000 	struct reloc *reloc;
4001 	unsigned long off;
4002 	int warnings = 0;
4003 
4004 	/*
4005 	 * Looking for function pointer load relocations.  Ignore
4006 	 * direct/indirect branches:
4007 	 */
4008 	switch (insn->type) {
4009 	case INSN_CALL:
4010 	case INSN_CALL_DYNAMIC:
4011 	case INSN_JUMP_CONDITIONAL:
4012 	case INSN_JUMP_UNCONDITIONAL:
4013 	case INSN_JUMP_DYNAMIC:
4014 	case INSN_JUMP_DYNAMIC_CONDITIONAL:
4015 	case INSN_RETURN:
4016 	case INSN_NOP:
4017 		return 0;
4018 	default:
4019 		break;
4020 	}
4021 
4022 	for (reloc = insn_reloc(file, insn);
4023 	     reloc;
4024 	     reloc = find_reloc_by_dest_range(file->elf, insn->sec,
4025 					      reloc->offset + 1,
4026 					      (insn->offset + insn->len) - (reloc->offset + 1))) {
4027 
4028 		/*
4029 		 * static_call_update() references the trampoline, which
4030 		 * doesn't have (or need) ENDBR.  Skip warning in that case.
4031 		 */
4032 		if (reloc->sym->static_call_tramp)
4033 			continue;
4034 
4035 		off = reloc->sym->offset;
4036 		if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
4037 			off += arch_dest_reloc_offset(reloc->addend);
4038 		else
4039 			off += reloc->addend;
4040 
4041 		dest = find_insn(file, reloc->sym->sec, off);
4042 		if (!dest)
4043 			continue;
4044 
4045 		if (dest->type == INSN_ENDBR) {
4046 			mark_endbr_used(dest);
4047 			continue;
4048 		}
4049 
4050 		if (dest->func && dest->func == insn->func) {
4051 			/*
4052 			 * Anything from->to self is either _THIS_IP_ or
4053 			 * IRET-to-self.
4054 			 *
4055 			 * There is no sane way to annotate _THIS_IP_ since the
4056 			 * compiler treats the relocation as a constant and is
4057 			 * happy to fold in offsets, skewing any annotation we
4058 			 * do, leading to vast amounts of false-positives.
4059 			 *
4060 			 * There's also compiler generated _THIS_IP_ through
4061 			 * KCOV and such which we have no hope of annotating.
4062 			 *
4063 			 * As such, blanket accept self-references without
4064 			 * issue.
4065 			 */
4066 			continue;
4067 		}
4068 
4069 		if (dest->noendbr)
4070 			continue;
4071 
4072 		WARN_FUNC("relocation to !ENDBR: %s",
4073 			  insn->sec, insn->offset,
4074 			  offstr(dest->sec, dest->offset));
4075 
4076 		warnings++;
4077 	}
4078 
4079 	return warnings;
4080 }
4081 
validate_ibt_data_reloc(struct objtool_file * file,struct reloc * reloc)4082 static int validate_ibt_data_reloc(struct objtool_file *file,
4083 				   struct reloc *reloc)
4084 {
4085 	struct instruction *dest;
4086 
4087 	dest = find_insn(file, reloc->sym->sec,
4088 			 reloc->sym->offset + reloc->addend);
4089 	if (!dest)
4090 		return 0;
4091 
4092 	if (dest->type == INSN_ENDBR) {
4093 		mark_endbr_used(dest);
4094 		return 0;
4095 	}
4096 
4097 	if (dest->noendbr)
4098 		return 0;
4099 
4100 	WARN_FUNC("data relocation to !ENDBR: %s",
4101 		  reloc->sec->base, reloc->offset,
4102 		  offstr(dest->sec, dest->offset));
4103 
4104 	return 1;
4105 }
4106 
4107 /*
4108  * Validate IBT rules and remove used ENDBR instructions from the seal list.
4109  * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with
4110  * NOPs) later, in create_ibt_endbr_seal_sections().
4111  */
validate_ibt(struct objtool_file * file)4112 static int validate_ibt(struct objtool_file *file)
4113 {
4114 	struct section *sec;
4115 	struct reloc *reloc;
4116 	struct instruction *insn;
4117 	int warnings = 0;
4118 
4119 	for_each_insn(file, insn)
4120 		warnings += validate_ibt_insn(file, insn);
4121 
4122 	for_each_sec(file, sec) {
4123 
4124 		/* Already done by validate_ibt_insn() */
4125 		if (sec->sh.sh_flags & SHF_EXECINSTR)
4126 			continue;
4127 
4128 		if (!sec->reloc)
4129 			continue;
4130 
4131 		/*
4132 		 * These sections can reference text addresses, but not with
4133 		 * the intent to indirect branch to them.
4134 		 */
4135 		if ((!strncmp(sec->name, ".discard", 8) &&
4136 		     strcmp(sec->name, ".discard.ibt_endbr_noseal"))	||
4137 		    !strncmp(sec->name, ".debug", 6)			||
4138 		    !strcmp(sec->name, ".altinstructions")		||
4139 		    !strcmp(sec->name, ".ibt_endbr_seal")		||
4140 		    !strcmp(sec->name, ".orc_unwind_ip")		||
4141 		    !strcmp(sec->name, ".parainstructions")		||
4142 		    !strcmp(sec->name, ".retpoline_sites")		||
4143 		    !strcmp(sec->name, ".smp_locks")			||
4144 		    !strcmp(sec->name, ".static_call_sites")		||
4145 		    !strcmp(sec->name, "_error_injection_whitelist")	||
4146 		    !strcmp(sec->name, "_kprobe_blacklist")		||
4147 		    !strcmp(sec->name, "__bug_table")			||
4148 		    !strcmp(sec->name, "__ex_table")			||
4149 		    !strcmp(sec->name, "__jump_table")			||
4150 		    !strcmp(sec->name, "__mcount_loc")			||
4151 		    !strcmp(sec->name, ".kcfi_traps")			||
4152 		    strstr(sec->name, "__patchable_function_entries"))
4153 			continue;
4154 
4155 		list_for_each_entry(reloc, &sec->reloc->reloc_list, list)
4156 			warnings += validate_ibt_data_reloc(file, reloc);
4157 	}
4158 
4159 	return warnings;
4160 }
4161 
validate_sls(struct objtool_file * file)4162 static int validate_sls(struct objtool_file *file)
4163 {
4164 	struct instruction *insn, *next_insn;
4165 	int warnings = 0;
4166 
4167 	for_each_insn(file, insn) {
4168 		next_insn = next_insn_same_sec(file, insn);
4169 
4170 		if (insn->retpoline_safe)
4171 			continue;
4172 
4173 		switch (insn->type) {
4174 		case INSN_RETURN:
4175 			if (!next_insn || next_insn->type != INSN_TRAP) {
4176 				WARN_FUNC("missing int3 after ret",
4177 					  insn->sec, insn->offset);
4178 				warnings++;
4179 			}
4180 
4181 			break;
4182 		case INSN_JUMP_DYNAMIC:
4183 			if (!next_insn || next_insn->type != INSN_TRAP) {
4184 				WARN_FUNC("missing int3 after indirect jump",
4185 					  insn->sec, insn->offset);
4186 				warnings++;
4187 			}
4188 			break;
4189 		default:
4190 			break;
4191 		}
4192 	}
4193 
4194 	return warnings;
4195 }
4196 
validate_reachable_instructions(struct objtool_file * file)4197 static int validate_reachable_instructions(struct objtool_file *file)
4198 {
4199 	struct instruction *insn;
4200 
4201 	if (file->ignore_unreachables)
4202 		return 0;
4203 
4204 	for_each_insn(file, insn) {
4205 		if (insn->visited || ignore_unreachable_insn(file, insn))
4206 			continue;
4207 
4208 		WARN_FUNC("unreachable instruction", insn->sec, insn->offset);
4209 		return 1;
4210 	}
4211 
4212 	return 0;
4213 }
4214 
check(struct objtool_file * file)4215 int check(struct objtool_file *file)
4216 {
4217 	int ret, warnings = 0;
4218 
4219 	arch_initial_func_cfi_state(&initial_func_cfi);
4220 	init_cfi_state(&init_cfi);
4221 	init_cfi_state(&func_cfi);
4222 	set_func_state(&func_cfi);
4223 
4224 	if (!cfi_hash_alloc(1UL << (file->elf->symbol_bits - 3)))
4225 		goto out;
4226 
4227 	cfi_hash_add(&init_cfi);
4228 	cfi_hash_add(&func_cfi);
4229 
4230 	ret = decode_sections(file);
4231 	if (ret < 0)
4232 		goto out;
4233 
4234 	warnings += ret;
4235 
4236 	if (list_empty(&file->insn_list))
4237 		goto out;
4238 
4239 	if (opts.retpoline) {
4240 		ret = validate_retpoline(file);
4241 		if (ret < 0)
4242 			return ret;
4243 		warnings += ret;
4244 	}
4245 
4246 	if (opts.stackval || opts.orc || opts.uaccess) {
4247 		ret = validate_functions(file);
4248 		if (ret < 0)
4249 			goto out;
4250 		warnings += ret;
4251 
4252 		ret = validate_unwind_hints(file, NULL);
4253 		if (ret < 0)
4254 			goto out;
4255 		warnings += ret;
4256 
4257 		if (!warnings) {
4258 			ret = validate_reachable_instructions(file);
4259 			if (ret < 0)
4260 				goto out;
4261 			warnings += ret;
4262 		}
4263 
4264 	} else if (opts.noinstr) {
4265 		ret = validate_noinstr_sections(file);
4266 		if (ret < 0)
4267 			goto out;
4268 		warnings += ret;
4269 	}
4270 
4271 	if (opts.unret) {
4272 		/*
4273 		 * Must be after validate_branch() and friends, it plays
4274 		 * further games with insn->visited.
4275 		 */
4276 		ret = validate_unret(file);
4277 		if (ret < 0)
4278 			return ret;
4279 		warnings += ret;
4280 	}
4281 
4282 	if (opts.ibt) {
4283 		ret = validate_ibt(file);
4284 		if (ret < 0)
4285 			goto out;
4286 		warnings += ret;
4287 	}
4288 
4289 	if (opts.sls) {
4290 		ret = validate_sls(file);
4291 		if (ret < 0)
4292 			goto out;
4293 		warnings += ret;
4294 	}
4295 
4296 	if (opts.static_call) {
4297 		ret = create_static_call_sections(file);
4298 		if (ret < 0)
4299 			goto out;
4300 		warnings += ret;
4301 	}
4302 
4303 	if (opts.retpoline) {
4304 		ret = create_retpoline_sites_sections(file);
4305 		if (ret < 0)
4306 			goto out;
4307 		warnings += ret;
4308 	}
4309 
4310 	if (opts.rethunk) {
4311 		ret = create_return_sites_sections(file);
4312 		if (ret < 0)
4313 			goto out;
4314 		warnings += ret;
4315 	}
4316 
4317 	if (opts.mcount) {
4318 		ret = create_mcount_loc_sections(file);
4319 		if (ret < 0)
4320 			goto out;
4321 		warnings += ret;
4322 	}
4323 
4324 	if (opts.ibt) {
4325 		ret = create_ibt_endbr_seal_sections(file);
4326 		if (ret < 0)
4327 			goto out;
4328 		warnings += ret;
4329 	}
4330 
4331 	if (opts.orc && !list_empty(&file->insn_list)) {
4332 		ret = orc_create(file);
4333 		if (ret < 0)
4334 			goto out;
4335 		warnings += ret;
4336 	}
4337 
4338 
4339 	if (opts.stats) {
4340 		printf("nr_insns_visited: %ld\n", nr_insns_visited);
4341 		printf("nr_cfi: %ld\n", nr_cfi);
4342 		printf("nr_cfi_reused: %ld\n", nr_cfi_reused);
4343 		printf("nr_cfi_cache: %ld\n", nr_cfi_cache);
4344 	}
4345 
4346 out:
4347 	/*
4348 	 *  For now, don't fail the kernel build on fatal warnings.  These
4349 	 *  errors are still fairly common due to the growing matrix of
4350 	 *  supported toolchains and their recent pace of change.
4351 	 */
4352 	return 0;
4353 }
4354