1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Load ELF vmlinux file for the kexec_file_load syscall.
4 *
5 * Copyright (C) 2021 Huawei Technologies Co, Ltd.
6 *
7 * Author: Liao Chang (liaochang1@huawei.com)
8 *
9 * Based on kexec-tools' kexec-elf-riscv.c, heavily modified
10 * for kernel.
11 */
12
13 #define pr_fmt(fmt) "kexec_image: " fmt
14
15 #include <linux/elf.h>
16 #include <linux/kexec.h>
17 #include <linux/slab.h>
18 #include <linux/of.h>
19 #include <linux/libfdt.h>
20 #include <linux/types.h>
21 #include <linux/memblock.h>
22 #include <asm/setup.h>
23
arch_kimage_file_post_load_cleanup(struct kimage * image)24 int arch_kimage_file_post_load_cleanup(struct kimage *image)
25 {
26 kvfree(image->arch.fdt);
27 image->arch.fdt = NULL;
28
29 vfree(image->elf_headers);
30 image->elf_headers = NULL;
31 image->elf_headers_sz = 0;
32
33 return kexec_image_post_load_cleanup_default(image);
34 }
35
riscv_kexec_elf_load(struct kimage * image,struct elfhdr * ehdr,struct kexec_elf_info * elf_info,unsigned long old_pbase,unsigned long new_pbase)36 static int riscv_kexec_elf_load(struct kimage *image, struct elfhdr *ehdr,
37 struct kexec_elf_info *elf_info, unsigned long old_pbase,
38 unsigned long new_pbase)
39 {
40 int i;
41 int ret = 0;
42 size_t size;
43 struct kexec_buf kbuf;
44 const struct elf_phdr *phdr;
45
46 kbuf.image = image;
47
48 for (i = 0; i < ehdr->e_phnum; i++) {
49 phdr = &elf_info->proghdrs[i];
50 if (phdr->p_type != PT_LOAD)
51 continue;
52
53 size = phdr->p_filesz;
54 if (size > phdr->p_memsz)
55 size = phdr->p_memsz;
56
57 kbuf.buffer = (void *) elf_info->buffer + phdr->p_offset;
58 kbuf.bufsz = size;
59 kbuf.buf_align = phdr->p_align;
60 kbuf.mem = phdr->p_paddr - old_pbase + new_pbase;
61 kbuf.memsz = phdr->p_memsz;
62 kbuf.top_down = false;
63 ret = kexec_add_buffer(&kbuf);
64 if (ret)
65 break;
66 }
67
68 return ret;
69 }
70
71 /*
72 * Go through the available phsyical memory regions and find one that hold
73 * an image of the specified size.
74 */
elf_find_pbase(struct kimage * image,unsigned long kernel_len,struct elfhdr * ehdr,struct kexec_elf_info * elf_info,unsigned long * old_pbase,unsigned long * new_pbase)75 static int elf_find_pbase(struct kimage *image, unsigned long kernel_len,
76 struct elfhdr *ehdr, struct kexec_elf_info *elf_info,
77 unsigned long *old_pbase, unsigned long *new_pbase)
78 {
79 int i;
80 int ret;
81 struct kexec_buf kbuf;
82 const struct elf_phdr *phdr;
83 unsigned long lowest_paddr = ULONG_MAX;
84 unsigned long lowest_vaddr = ULONG_MAX;
85
86 for (i = 0; i < ehdr->e_phnum; i++) {
87 phdr = &elf_info->proghdrs[i];
88 if (phdr->p_type != PT_LOAD)
89 continue;
90
91 if (lowest_paddr > phdr->p_paddr)
92 lowest_paddr = phdr->p_paddr;
93
94 if (lowest_vaddr > phdr->p_vaddr)
95 lowest_vaddr = phdr->p_vaddr;
96 }
97
98 kbuf.image = image;
99 kbuf.buf_min = lowest_paddr;
100 kbuf.buf_max = ULONG_MAX;
101 kbuf.buf_align = PAGE_SIZE;
102 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
103 kbuf.memsz = ALIGN(kernel_len, PAGE_SIZE);
104 kbuf.top_down = false;
105 ret = arch_kexec_locate_mem_hole(&kbuf);
106 if (!ret) {
107 *old_pbase = lowest_paddr;
108 *new_pbase = kbuf.mem;
109 image->start = ehdr->e_entry - lowest_vaddr + kbuf.mem;
110 }
111 return ret;
112 }
113
get_nr_ram_ranges_callback(struct resource * res,void * arg)114 static int get_nr_ram_ranges_callback(struct resource *res, void *arg)
115 {
116 unsigned int *nr_ranges = arg;
117
118 (*nr_ranges)++;
119 return 0;
120 }
121
prepare_elf64_ram_headers_callback(struct resource * res,void * arg)122 static int prepare_elf64_ram_headers_callback(struct resource *res, void *arg)
123 {
124 struct crash_mem *cmem = arg;
125
126 cmem->ranges[cmem->nr_ranges].start = res->start;
127 cmem->ranges[cmem->nr_ranges].end = res->end;
128 cmem->nr_ranges++;
129
130 return 0;
131 }
132
prepare_elf_headers(void ** addr,unsigned long * sz)133 static int prepare_elf_headers(void **addr, unsigned long *sz)
134 {
135 struct crash_mem *cmem;
136 unsigned int nr_ranges;
137 int ret;
138
139 nr_ranges = 1; /* For exclusion of crashkernel region */
140 walk_system_ram_res(0, -1, &nr_ranges, get_nr_ram_ranges_callback);
141
142 cmem = kmalloc(struct_size(cmem, ranges, nr_ranges), GFP_KERNEL);
143 if (!cmem)
144 return -ENOMEM;
145
146 cmem->max_nr_ranges = nr_ranges;
147 cmem->nr_ranges = 0;
148 ret = walk_system_ram_res(0, -1, cmem, prepare_elf64_ram_headers_callback);
149 if (ret)
150 goto out;
151
152 /* Exclude crashkernel region */
153 ret = crash_exclude_mem_range(cmem, crashk_res.start, crashk_res.end);
154 if (!ret)
155 ret = crash_prepare_elf64_headers(cmem, true, addr, sz);
156
157 out:
158 kfree(cmem);
159 return ret;
160 }
161
setup_kdump_cmdline(struct kimage * image,char * cmdline,unsigned long cmdline_len)162 static char *setup_kdump_cmdline(struct kimage *image, char *cmdline,
163 unsigned long cmdline_len)
164 {
165 int elfcorehdr_strlen;
166 char *cmdline_ptr;
167
168 cmdline_ptr = kzalloc(COMMAND_LINE_SIZE, GFP_KERNEL);
169 if (!cmdline_ptr)
170 return NULL;
171
172 elfcorehdr_strlen = sprintf(cmdline_ptr, "elfcorehdr=0x%lx ",
173 image->elf_load_addr);
174
175 if (elfcorehdr_strlen + cmdline_len > COMMAND_LINE_SIZE) {
176 pr_err("Appending elfcorehdr=<addr> exceeds cmdline size\n");
177 kfree(cmdline_ptr);
178 return NULL;
179 }
180
181 memcpy(cmdline_ptr + elfcorehdr_strlen, cmdline, cmdline_len);
182 /* Ensure it's nul terminated */
183 cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0';
184 return cmdline_ptr;
185 }
186
elf_kexec_load(struct kimage * image,char * kernel_buf,unsigned long kernel_len,char * initrd,unsigned long initrd_len,char * cmdline,unsigned long cmdline_len)187 static void *elf_kexec_load(struct kimage *image, char *kernel_buf,
188 unsigned long kernel_len, char *initrd,
189 unsigned long initrd_len, char *cmdline,
190 unsigned long cmdline_len)
191 {
192 int ret;
193 unsigned long old_kernel_pbase = ULONG_MAX;
194 unsigned long new_kernel_pbase = 0UL;
195 unsigned long initrd_pbase = 0UL;
196 unsigned long headers_sz;
197 unsigned long kernel_start;
198 void *fdt, *headers;
199 struct elfhdr ehdr;
200 struct kexec_buf kbuf;
201 struct kexec_elf_info elf_info;
202 char *modified_cmdline = NULL;
203
204 ret = kexec_build_elf_info(kernel_buf, kernel_len, &ehdr, &elf_info);
205 if (ret)
206 return ERR_PTR(ret);
207
208 ret = elf_find_pbase(image, kernel_len, &ehdr, &elf_info,
209 &old_kernel_pbase, &new_kernel_pbase);
210 if (ret)
211 goto out;
212 kernel_start = image->start;
213 pr_notice("The entry point of kernel at 0x%lx\n", image->start);
214
215 /* Add the kernel binary to the image */
216 ret = riscv_kexec_elf_load(image, &ehdr, &elf_info,
217 old_kernel_pbase, new_kernel_pbase);
218 if (ret)
219 goto out;
220
221 kbuf.image = image;
222 kbuf.buf_min = new_kernel_pbase + kernel_len;
223 kbuf.buf_max = ULONG_MAX;
224
225 /* Add elfcorehdr */
226 if (image->type == KEXEC_TYPE_CRASH) {
227 ret = prepare_elf_headers(&headers, &headers_sz);
228 if (ret) {
229 pr_err("Preparing elf core header failed\n");
230 goto out;
231 }
232
233 kbuf.buffer = headers;
234 kbuf.bufsz = headers_sz;
235 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
236 kbuf.memsz = headers_sz;
237 kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
238 kbuf.top_down = true;
239
240 ret = kexec_add_buffer(&kbuf);
241 if (ret) {
242 vfree(headers);
243 goto out;
244 }
245 image->elf_headers = headers;
246 image->elf_load_addr = kbuf.mem;
247 image->elf_headers_sz = headers_sz;
248
249 pr_debug("Loaded elf core header at 0x%lx bufsz=0x%lx memsz=0x%lx\n",
250 image->elf_load_addr, kbuf.bufsz, kbuf.memsz);
251
252 /* Setup cmdline for kdump kernel case */
253 modified_cmdline = setup_kdump_cmdline(image, cmdline,
254 cmdline_len);
255 if (!modified_cmdline) {
256 pr_err("Setting up cmdline for kdump kernel failed\n");
257 ret = -EINVAL;
258 goto out;
259 }
260 cmdline = modified_cmdline;
261 }
262
263 #ifdef CONFIG_ARCH_HAS_KEXEC_PURGATORY
264 /* Add purgatory to the image */
265 kbuf.top_down = true;
266 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
267 ret = kexec_load_purgatory(image, &kbuf);
268 if (ret) {
269 pr_err("Error loading purgatory ret=%d\n", ret);
270 goto out;
271 }
272 ret = kexec_purgatory_get_set_symbol(image, "riscv_kernel_entry",
273 &kernel_start,
274 sizeof(kernel_start), 0);
275 if (ret)
276 pr_err("Error update purgatory ret=%d\n", ret);
277 #endif /* CONFIG_ARCH_HAS_KEXEC_PURGATORY */
278
279 /* Add the initrd to the image */
280 if (initrd != NULL) {
281 kbuf.buffer = initrd;
282 kbuf.bufsz = kbuf.memsz = initrd_len;
283 kbuf.buf_align = PAGE_SIZE;
284 kbuf.top_down = false;
285 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
286 ret = kexec_add_buffer(&kbuf);
287 if (ret)
288 goto out;
289 initrd_pbase = kbuf.mem;
290 pr_notice("Loaded initrd at 0x%lx\n", initrd_pbase);
291 }
292
293 /* Add the DTB to the image */
294 fdt = of_kexec_alloc_and_setup_fdt(image, initrd_pbase,
295 initrd_len, cmdline, 0);
296 if (!fdt) {
297 pr_err("Error setting up the new device tree.\n");
298 ret = -EINVAL;
299 goto out;
300 }
301
302 fdt_pack(fdt);
303 kbuf.buffer = fdt;
304 kbuf.bufsz = kbuf.memsz = fdt_totalsize(fdt);
305 kbuf.buf_align = PAGE_SIZE;
306 kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
307 kbuf.top_down = true;
308 ret = kexec_add_buffer(&kbuf);
309 if (ret) {
310 pr_err("Error add DTB kbuf ret=%d\n", ret);
311 goto out_free_fdt;
312 }
313 /* Cache the fdt buffer address for memory cleanup */
314 image->arch.fdt = fdt;
315 pr_notice("Loaded device tree at 0x%lx\n", kbuf.mem);
316 goto out;
317
318 out_free_fdt:
319 kvfree(fdt);
320 out:
321 kfree(modified_cmdline);
322 kexec_free_elf_info(&elf_info);
323 return ret ? ERR_PTR(ret) : NULL;
324 }
325
326 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
327 #define RISCV_IMM_BITS 12
328 #define RISCV_IMM_REACH (1LL << RISCV_IMM_BITS)
329 #define RISCV_CONST_HIGH_PART(x) \
330 (((x) + (RISCV_IMM_REACH >> 1)) & ~(RISCV_IMM_REACH - 1))
331 #define RISCV_CONST_LOW_PART(x) ((x) - RISCV_CONST_HIGH_PART(x))
332
333 #define ENCODE_ITYPE_IMM(x) \
334 (RV_X(x, 0, 12) << 20)
335 #define ENCODE_BTYPE_IMM(x) \
336 ((RV_X(x, 1, 4) << 8) | (RV_X(x, 5, 6) << 25) | \
337 (RV_X(x, 11, 1) << 7) | (RV_X(x, 12, 1) << 31))
338 #define ENCODE_UTYPE_IMM(x) \
339 (RV_X(x, 12, 20) << 12)
340 #define ENCODE_JTYPE_IMM(x) \
341 ((RV_X(x, 1, 10) << 21) | (RV_X(x, 11, 1) << 20) | \
342 (RV_X(x, 12, 8) << 12) | (RV_X(x, 20, 1) << 31))
343 #define ENCODE_CBTYPE_IMM(x) \
344 ((RV_X(x, 1, 2) << 3) | (RV_X(x, 3, 2) << 10) | (RV_X(x, 5, 1) << 2) | \
345 (RV_X(x, 6, 2) << 5) | (RV_X(x, 8, 1) << 12))
346 #define ENCODE_CJTYPE_IMM(x) \
347 ((RV_X(x, 1, 3) << 3) | (RV_X(x, 4, 1) << 11) | (RV_X(x, 5, 1) << 2) | \
348 (RV_X(x, 6, 1) << 7) | (RV_X(x, 7, 1) << 6) | (RV_X(x, 8, 2) << 9) | \
349 (RV_X(x, 10, 1) << 8) | (RV_X(x, 11, 1) << 12))
350 #define ENCODE_UJTYPE_IMM(x) \
351 (ENCODE_UTYPE_IMM(RISCV_CONST_HIGH_PART(x)) | \
352 (ENCODE_ITYPE_IMM(RISCV_CONST_LOW_PART(x)) << 32))
353 #define ENCODE_UITYPE_IMM(x) \
354 (ENCODE_UTYPE_IMM(x) | (ENCODE_ITYPE_IMM(x) << 32))
355
356 #define CLEAN_IMM(type, x) \
357 ((~ENCODE_##type##_IMM((uint64_t)(-1))) & (x))
358
arch_kexec_apply_relocations_add(struct purgatory_info * pi,Elf_Shdr * section,const Elf_Shdr * relsec,const Elf_Shdr * symtab)359 int arch_kexec_apply_relocations_add(struct purgatory_info *pi,
360 Elf_Shdr *section,
361 const Elf_Shdr *relsec,
362 const Elf_Shdr *symtab)
363 {
364 const char *strtab, *name, *shstrtab;
365 const Elf_Shdr *sechdrs;
366 Elf64_Rela *relas;
367 int i, r_type;
368
369 /* String & section header string table */
370 sechdrs = (void *)pi->ehdr + pi->ehdr->e_shoff;
371 strtab = (char *)pi->ehdr + sechdrs[symtab->sh_link].sh_offset;
372 shstrtab = (char *)pi->ehdr + sechdrs[pi->ehdr->e_shstrndx].sh_offset;
373
374 relas = (void *)pi->ehdr + relsec->sh_offset;
375
376 for (i = 0; i < relsec->sh_size / sizeof(*relas); i++) {
377 const Elf_Sym *sym; /* symbol to relocate */
378 unsigned long addr; /* final location after relocation */
379 unsigned long val; /* relocated symbol value */
380 unsigned long sec_base; /* relocated symbol value */
381 void *loc; /* tmp location to modify */
382
383 sym = (void *)pi->ehdr + symtab->sh_offset;
384 sym += ELF64_R_SYM(relas[i].r_info);
385
386 if (sym->st_name)
387 name = strtab + sym->st_name;
388 else
389 name = shstrtab + sechdrs[sym->st_shndx].sh_name;
390
391 loc = pi->purgatory_buf;
392 loc += section->sh_offset;
393 loc += relas[i].r_offset;
394
395 if (sym->st_shndx == SHN_ABS)
396 sec_base = 0;
397 else if (sym->st_shndx >= pi->ehdr->e_shnum) {
398 pr_err("Invalid section %d for symbol %s\n",
399 sym->st_shndx, name);
400 return -ENOEXEC;
401 } else
402 sec_base = pi->sechdrs[sym->st_shndx].sh_addr;
403
404 val = sym->st_value;
405 val += sec_base;
406 val += relas[i].r_addend;
407
408 addr = section->sh_addr + relas[i].r_offset;
409
410 r_type = ELF64_R_TYPE(relas[i].r_info);
411
412 switch (r_type) {
413 case R_RISCV_BRANCH:
414 *(u32 *)loc = CLEAN_IMM(BTYPE, *(u32 *)loc) |
415 ENCODE_BTYPE_IMM(val - addr);
416 break;
417 case R_RISCV_JAL:
418 *(u32 *)loc = CLEAN_IMM(JTYPE, *(u32 *)loc) |
419 ENCODE_JTYPE_IMM(val - addr);
420 break;
421 /*
422 * With no R_RISCV_PCREL_LO12_S, R_RISCV_PCREL_LO12_I
423 * sym is expected to be next to R_RISCV_PCREL_HI20
424 * in purgatory relsec. Handle it like R_RISCV_CALL
425 * sym, instead of searching the whole relsec.
426 */
427 case R_RISCV_PCREL_HI20:
428 case R_RISCV_CALL:
429 *(u64 *)loc = CLEAN_IMM(UITYPE, *(u64 *)loc) |
430 ENCODE_UJTYPE_IMM(val - addr);
431 break;
432 case R_RISCV_RVC_BRANCH:
433 *(u32 *)loc = CLEAN_IMM(CBTYPE, *(u32 *)loc) |
434 ENCODE_CBTYPE_IMM(val - addr);
435 break;
436 case R_RISCV_RVC_JUMP:
437 *(u32 *)loc = CLEAN_IMM(CJTYPE, *(u32 *)loc) |
438 ENCODE_CJTYPE_IMM(val - addr);
439 break;
440 case R_RISCV_ADD32:
441 *(u32 *)loc += val;
442 break;
443 case R_RISCV_SUB32:
444 *(u32 *)loc -= val;
445 break;
446 /* It has been applied by R_RISCV_PCREL_HI20 sym */
447 case R_RISCV_PCREL_LO12_I:
448 case R_RISCV_ALIGN:
449 case R_RISCV_RELAX:
450 break;
451 default:
452 pr_err("Unknown rela relocation: %d\n", r_type);
453 return -ENOEXEC;
454 }
455 }
456 return 0;
457 }
458
459 const struct kexec_file_ops elf_kexec_ops = {
460 .probe = kexec_elf_probe,
461 .load = elf_kexec_load,
462 };
463