1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Copyright 2008 Michael Ellerman, IBM Corporation.
4  */
5 
6 #include <linux/kprobes.h>
7 #include <linux/vmalloc.h>
8 #include <linux/init.h>
9 #include <linux/cpuhotplug.h>
10 #include <linux/uaccess.h>
11 #include <linux/jump_label.h>
12 
13 #include <asm/tlbflush.h>
14 #include <asm/page.h>
15 #include <asm/code-patching.h>
16 #include <asm/inst.h>
17 
__patch_instruction(u32 * exec_addr,ppc_inst_t instr,u32 * patch_addr)18 static int __patch_instruction(u32 *exec_addr, ppc_inst_t instr, u32 *patch_addr)
19 {
20 	if (!ppc_inst_prefixed(instr)) {
21 		u32 val = ppc_inst_val(instr);
22 
23 		__put_kernel_nofault(patch_addr, &val, u32, failed);
24 	} else {
25 		u64 val = ppc_inst_as_ulong(instr);
26 
27 		__put_kernel_nofault(patch_addr, &val, u64, failed);
28 	}
29 
30 	asm ("dcbst 0, %0; sync; icbi 0,%1; sync; isync" :: "r" (patch_addr),
31 							    "r" (exec_addr));
32 
33 	return 0;
34 
35 failed:
36 	return -EPERM;
37 }
38 
raw_patch_instruction(u32 * addr,ppc_inst_t instr)39 int raw_patch_instruction(u32 *addr, ppc_inst_t instr)
40 {
41 	return __patch_instruction(addr, instr, addr);
42 }
43 
44 #ifdef CONFIG_STRICT_KERNEL_RWX
45 static DEFINE_PER_CPU(struct vm_struct *, text_poke_area);
46 
47 static int map_patch_area(void *addr, unsigned long text_poke_addr);
48 static void unmap_patch_area(unsigned long addr);
49 
text_area_cpu_up(unsigned int cpu)50 static int text_area_cpu_up(unsigned int cpu)
51 {
52 	struct vm_struct *area;
53 	unsigned long addr;
54 	int err;
55 
56 	area = get_vm_area(PAGE_SIZE, VM_ALLOC);
57 	if (!area) {
58 		WARN_ONCE(1, "Failed to create text area for cpu %d\n",
59 			cpu);
60 		return -1;
61 	}
62 
63 	// Map/unmap the area to ensure all page tables are pre-allocated
64 	addr = (unsigned long)area->addr;
65 	err = map_patch_area(empty_zero_page, addr);
66 	if (err)
67 		return err;
68 
69 	unmap_patch_area(addr);
70 
71 	this_cpu_write(text_poke_area, area);
72 
73 	return 0;
74 }
75 
text_area_cpu_down(unsigned int cpu)76 static int text_area_cpu_down(unsigned int cpu)
77 {
78 	free_vm_area(this_cpu_read(text_poke_area));
79 	return 0;
80 }
81 
82 static __ro_after_init DEFINE_STATIC_KEY_FALSE(poking_init_done);
83 
84 /*
85  * Although BUG_ON() is rude, in this case it should only happen if ENOMEM, and
86  * we judge it as being preferable to a kernel that will crash later when
87  * someone tries to use patch_instruction().
88  */
poking_init(void)89 void __init poking_init(void)
90 {
91 	BUG_ON(!cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
92 		"powerpc/text_poke:online", text_area_cpu_up,
93 		text_area_cpu_down));
94 	static_branch_enable(&poking_init_done);
95 }
96 
get_patch_pfn(void * addr)97 static unsigned long get_patch_pfn(void *addr)
98 {
99 	if (IS_ENABLED(CONFIG_MODULES) && is_vmalloc_or_module_addr(addr))
100 		return vmalloc_to_pfn(addr);
101 	else
102 		return __pa_symbol(addr) >> PAGE_SHIFT;
103 }
104 
105 /*
106  * This can be called for kernel text or a module.
107  */
map_patch_area(void * addr,unsigned long text_poke_addr)108 static int map_patch_area(void *addr, unsigned long text_poke_addr)
109 {
110 	unsigned long pfn = get_patch_pfn(addr);
111 
112 	return map_kernel_page(text_poke_addr, (pfn << PAGE_SHIFT), PAGE_KERNEL);
113 }
114 
unmap_patch_area(unsigned long addr)115 static void unmap_patch_area(unsigned long addr)
116 {
117 	pte_t *ptep;
118 	pmd_t *pmdp;
119 	pud_t *pudp;
120 	p4d_t *p4dp;
121 	pgd_t *pgdp;
122 
123 	pgdp = pgd_offset_k(addr);
124 	if (WARN_ON(pgd_none(*pgdp)))
125 		return;
126 
127 	p4dp = p4d_offset(pgdp, addr);
128 	if (WARN_ON(p4d_none(*p4dp)))
129 		return;
130 
131 	pudp = pud_offset(p4dp, addr);
132 	if (WARN_ON(pud_none(*pudp)))
133 		return;
134 
135 	pmdp = pmd_offset(pudp, addr);
136 	if (WARN_ON(pmd_none(*pmdp)))
137 		return;
138 
139 	ptep = pte_offset_kernel(pmdp, addr);
140 	if (WARN_ON(pte_none(*ptep)))
141 		return;
142 
143 	/*
144 	 * In hash, pte_clear flushes the tlb, in radix, we have to
145 	 */
146 	pte_clear(&init_mm, addr, ptep);
147 	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
148 }
149 
__do_patch_instruction(u32 * addr,ppc_inst_t instr)150 static int __do_patch_instruction(u32 *addr, ppc_inst_t instr)
151 {
152 	int err;
153 	u32 *patch_addr;
154 	unsigned long text_poke_addr;
155 	pte_t *pte;
156 	unsigned long pfn = get_patch_pfn(addr);
157 
158 	text_poke_addr = (unsigned long)__this_cpu_read(text_poke_area)->addr & PAGE_MASK;
159 	patch_addr = (u32 *)(text_poke_addr + offset_in_page(addr));
160 
161 	pte = virt_to_kpte(text_poke_addr);
162 	__set_pte_at(&init_mm, text_poke_addr, pte, pfn_pte(pfn, PAGE_KERNEL), 0);
163 	/* See ptesync comment in radix__set_pte_at() */
164 	if (radix_enabled())
165 		asm volatile("ptesync": : :"memory");
166 
167 	err = __patch_instruction(addr, instr, patch_addr);
168 
169 	pte_clear(&init_mm, text_poke_addr, pte);
170 	flush_tlb_kernel_range(text_poke_addr, text_poke_addr + PAGE_SIZE);
171 
172 	return err;
173 }
174 
do_patch_instruction(u32 * addr,ppc_inst_t instr)175 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
176 {
177 	int err;
178 	unsigned long flags;
179 
180 	/*
181 	 * During early early boot patch_instruction is called
182 	 * when text_poke_area is not ready, but we still need
183 	 * to allow patching. We just do the plain old patching
184 	 */
185 	if (!static_branch_likely(&poking_init_done))
186 		return raw_patch_instruction(addr, instr);
187 
188 	local_irq_save(flags);
189 	err = __do_patch_instruction(addr, instr);
190 	local_irq_restore(flags);
191 
192 	return err;
193 }
194 #else /* !CONFIG_STRICT_KERNEL_RWX */
195 
do_patch_instruction(u32 * addr,ppc_inst_t instr)196 static int do_patch_instruction(u32 *addr, ppc_inst_t instr)
197 {
198 	return raw_patch_instruction(addr, instr);
199 }
200 
201 #endif /* CONFIG_STRICT_KERNEL_RWX */
202 
203 __ro_after_init DEFINE_STATIC_KEY_FALSE(init_mem_is_free);
204 
patch_instruction(u32 * addr,ppc_inst_t instr)205 int patch_instruction(u32 *addr, ppc_inst_t instr)
206 {
207 	/* Make sure we aren't patching a freed init section */
208 	if (static_branch_likely(&init_mem_is_free) && init_section_contains(addr, 4))
209 		return 0;
210 
211 	return do_patch_instruction(addr, instr);
212 }
213 NOKPROBE_SYMBOL(patch_instruction);
214 
patch_branch(u32 * addr,unsigned long target,int flags)215 int patch_branch(u32 *addr, unsigned long target, int flags)
216 {
217 	ppc_inst_t instr;
218 
219 	if (create_branch(&instr, addr, target, flags))
220 		return -ERANGE;
221 
222 	return patch_instruction(addr, instr);
223 }
224 
225 /*
226  * Helper to check if a given instruction is a conditional branch
227  * Derived from the conditional checks in analyse_instr()
228  */
is_conditional_branch(ppc_inst_t instr)229 bool is_conditional_branch(ppc_inst_t instr)
230 {
231 	unsigned int opcode = ppc_inst_primary_opcode(instr);
232 
233 	if (opcode == 16)       /* bc, bca, bcl, bcla */
234 		return true;
235 	if (opcode == 19) {
236 		switch ((ppc_inst_val(instr) >> 1) & 0x3ff) {
237 		case 16:        /* bclr, bclrl */
238 		case 528:       /* bcctr, bcctrl */
239 		case 560:       /* bctar, bctarl */
240 			return true;
241 		}
242 	}
243 	return false;
244 }
245 NOKPROBE_SYMBOL(is_conditional_branch);
246 
create_cond_branch(ppc_inst_t * instr,const u32 * addr,unsigned long target,int flags)247 int create_cond_branch(ppc_inst_t *instr, const u32 *addr,
248 		       unsigned long target, int flags)
249 {
250 	long offset;
251 
252 	offset = target;
253 	if (! (flags & BRANCH_ABSOLUTE))
254 		offset = offset - (unsigned long)addr;
255 
256 	/* Check we can represent the target in the instruction format */
257 	if (!is_offset_in_cond_branch_range(offset))
258 		return 1;
259 
260 	/* Mask out the flags and target, so they don't step on each other. */
261 	*instr = ppc_inst(0x40000000 | (flags & 0x3FF0003) | (offset & 0xFFFC));
262 
263 	return 0;
264 }
265 
instr_is_relative_branch(ppc_inst_t instr)266 int instr_is_relative_branch(ppc_inst_t instr)
267 {
268 	if (ppc_inst_val(instr) & BRANCH_ABSOLUTE)
269 		return 0;
270 
271 	return instr_is_branch_iform(instr) || instr_is_branch_bform(instr);
272 }
273 
instr_is_relative_link_branch(ppc_inst_t instr)274 int instr_is_relative_link_branch(ppc_inst_t instr)
275 {
276 	return instr_is_relative_branch(instr) && (ppc_inst_val(instr) & BRANCH_SET_LINK);
277 }
278 
branch_iform_target(const u32 * instr)279 static unsigned long branch_iform_target(const u32 *instr)
280 {
281 	signed long imm;
282 
283 	imm = ppc_inst_val(ppc_inst_read(instr)) & 0x3FFFFFC;
284 
285 	/* If the top bit of the immediate value is set this is negative */
286 	if (imm & 0x2000000)
287 		imm -= 0x4000000;
288 
289 	if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
290 		imm += (unsigned long)instr;
291 
292 	return (unsigned long)imm;
293 }
294 
branch_bform_target(const u32 * instr)295 static unsigned long branch_bform_target(const u32 *instr)
296 {
297 	signed long imm;
298 
299 	imm = ppc_inst_val(ppc_inst_read(instr)) & 0xFFFC;
300 
301 	/* If the top bit of the immediate value is set this is negative */
302 	if (imm & 0x8000)
303 		imm -= 0x10000;
304 
305 	if ((ppc_inst_val(ppc_inst_read(instr)) & BRANCH_ABSOLUTE) == 0)
306 		imm += (unsigned long)instr;
307 
308 	return (unsigned long)imm;
309 }
310 
branch_target(const u32 * instr)311 unsigned long branch_target(const u32 *instr)
312 {
313 	if (instr_is_branch_iform(ppc_inst_read(instr)))
314 		return branch_iform_target(instr);
315 	else if (instr_is_branch_bform(ppc_inst_read(instr)))
316 		return branch_bform_target(instr);
317 
318 	return 0;
319 }
320 
translate_branch(ppc_inst_t * instr,const u32 * dest,const u32 * src)321 int translate_branch(ppc_inst_t *instr, const u32 *dest, const u32 *src)
322 {
323 	unsigned long target;
324 	target = branch_target(src);
325 
326 	if (instr_is_branch_iform(ppc_inst_read(src)))
327 		return create_branch(instr, dest, target,
328 				     ppc_inst_val(ppc_inst_read(src)));
329 	else if (instr_is_branch_bform(ppc_inst_read(src)))
330 		return create_cond_branch(instr, dest, target,
331 					  ppc_inst_val(ppc_inst_read(src)));
332 
333 	return 1;
334 }
335