1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013 Huawei Ltd.
4  * Author: Jiang Liu <liuj97@gmail.com>
5  *
6  * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
7  */
8 #include <linux/bitops.h>
9 #include <linux/bug.h>
10 #include <linux/printk.h>
11 #include <linux/sizes.h>
12 #include <linux/types.h>
13 
14 #include <asm/debug-monitors.h>
15 #include <asm/errno.h>
16 #include <asm/insn.h>
17 #include <asm/kprobes.h>
18 
19 #define AARCH64_INSN_SF_BIT	BIT(31)
20 #define AARCH64_INSN_N_BIT	BIT(22)
21 #define AARCH64_INSN_LSL_12	BIT(22)
22 
23 static const int aarch64_insn_encoding_class[] = {
24 	AARCH64_INSN_CLS_UNKNOWN,
25 	AARCH64_INSN_CLS_UNKNOWN,
26 	AARCH64_INSN_CLS_SVE,
27 	AARCH64_INSN_CLS_UNKNOWN,
28 	AARCH64_INSN_CLS_LDST,
29 	AARCH64_INSN_CLS_DP_REG,
30 	AARCH64_INSN_CLS_LDST,
31 	AARCH64_INSN_CLS_DP_FPSIMD,
32 	AARCH64_INSN_CLS_DP_IMM,
33 	AARCH64_INSN_CLS_DP_IMM,
34 	AARCH64_INSN_CLS_BR_SYS,
35 	AARCH64_INSN_CLS_BR_SYS,
36 	AARCH64_INSN_CLS_LDST,
37 	AARCH64_INSN_CLS_DP_REG,
38 	AARCH64_INSN_CLS_LDST,
39 	AARCH64_INSN_CLS_DP_FPSIMD,
40 };
41 
aarch64_get_insn_class(u32 insn)42 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
43 {
44 	return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
45 }
46 
aarch64_insn_is_steppable_hint(u32 insn)47 bool __kprobes aarch64_insn_is_steppable_hint(u32 insn)
48 {
49 	if (!aarch64_insn_is_hint(insn))
50 		return false;
51 
52 	switch (insn & 0xFE0) {
53 	case AARCH64_INSN_HINT_XPACLRI:
54 	case AARCH64_INSN_HINT_PACIA_1716:
55 	case AARCH64_INSN_HINT_PACIB_1716:
56 	case AARCH64_INSN_HINT_PACIAZ:
57 	case AARCH64_INSN_HINT_PACIASP:
58 	case AARCH64_INSN_HINT_PACIBZ:
59 	case AARCH64_INSN_HINT_PACIBSP:
60 	case AARCH64_INSN_HINT_BTI:
61 	case AARCH64_INSN_HINT_BTIC:
62 	case AARCH64_INSN_HINT_BTIJ:
63 	case AARCH64_INSN_HINT_BTIJC:
64 	case AARCH64_INSN_HINT_NOP:
65 		return true;
66 	default:
67 		return false;
68 	}
69 }
70 
aarch64_insn_is_branch_imm(u32 insn)71 bool aarch64_insn_is_branch_imm(u32 insn)
72 {
73 	return (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn) ||
74 		aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn) ||
75 		aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
76 		aarch64_insn_is_bcond(insn));
77 }
78 
aarch64_insn_uses_literal(u32 insn)79 bool __kprobes aarch64_insn_uses_literal(u32 insn)
80 {
81 	/* ldr/ldrsw (literal), prfm */
82 
83 	return aarch64_insn_is_ldr_lit(insn) ||
84 		aarch64_insn_is_ldrsw_lit(insn) ||
85 		aarch64_insn_is_adr_adrp(insn) ||
86 		aarch64_insn_is_prfm_lit(insn);
87 }
88 
aarch64_insn_is_branch(u32 insn)89 bool __kprobes aarch64_insn_is_branch(u32 insn)
90 {
91 	/* b, bl, cb*, tb*, ret*, b.cond, br*, blr* */
92 
93 	return aarch64_insn_is_b(insn) ||
94 		aarch64_insn_is_bl(insn) ||
95 		aarch64_insn_is_cbz(insn) ||
96 		aarch64_insn_is_cbnz(insn) ||
97 		aarch64_insn_is_tbz(insn) ||
98 		aarch64_insn_is_tbnz(insn) ||
99 		aarch64_insn_is_ret(insn) ||
100 		aarch64_insn_is_ret_auth(insn) ||
101 		aarch64_insn_is_br(insn) ||
102 		aarch64_insn_is_br_auth(insn) ||
103 		aarch64_insn_is_blr(insn) ||
104 		aarch64_insn_is_blr_auth(insn) ||
105 		aarch64_insn_is_bcond(insn);
106 }
107 
aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,u32 * maskp,int * shiftp)108 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
109 						u32 *maskp, int *shiftp)
110 {
111 	u32 mask;
112 	int shift;
113 
114 	switch (type) {
115 	case AARCH64_INSN_IMM_26:
116 		mask = BIT(26) - 1;
117 		shift = 0;
118 		break;
119 	case AARCH64_INSN_IMM_19:
120 		mask = BIT(19) - 1;
121 		shift = 5;
122 		break;
123 	case AARCH64_INSN_IMM_16:
124 		mask = BIT(16) - 1;
125 		shift = 5;
126 		break;
127 	case AARCH64_INSN_IMM_14:
128 		mask = BIT(14) - 1;
129 		shift = 5;
130 		break;
131 	case AARCH64_INSN_IMM_12:
132 		mask = BIT(12) - 1;
133 		shift = 10;
134 		break;
135 	case AARCH64_INSN_IMM_9:
136 		mask = BIT(9) - 1;
137 		shift = 12;
138 		break;
139 	case AARCH64_INSN_IMM_7:
140 		mask = BIT(7) - 1;
141 		shift = 15;
142 		break;
143 	case AARCH64_INSN_IMM_6:
144 	case AARCH64_INSN_IMM_S:
145 		mask = BIT(6) - 1;
146 		shift = 10;
147 		break;
148 	case AARCH64_INSN_IMM_R:
149 		mask = BIT(6) - 1;
150 		shift = 16;
151 		break;
152 	case AARCH64_INSN_IMM_N:
153 		mask = 1;
154 		shift = 22;
155 		break;
156 	default:
157 		return -EINVAL;
158 	}
159 
160 	*maskp = mask;
161 	*shiftp = shift;
162 
163 	return 0;
164 }
165 
166 #define ADR_IMM_HILOSPLIT	2
167 #define ADR_IMM_SIZE		SZ_2M
168 #define ADR_IMM_LOMASK		((1 << ADR_IMM_HILOSPLIT) - 1)
169 #define ADR_IMM_HIMASK		((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
170 #define ADR_IMM_LOSHIFT		29
171 #define ADR_IMM_HISHIFT		5
172 
aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type,u32 insn)173 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
174 {
175 	u32 immlo, immhi, mask;
176 	int shift;
177 
178 	switch (type) {
179 	case AARCH64_INSN_IMM_ADR:
180 		shift = 0;
181 		immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
182 		immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
183 		insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
184 		mask = ADR_IMM_SIZE - 1;
185 		break;
186 	default:
187 		if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
188 			pr_err("%s: unknown immediate encoding %d\n", __func__,
189 			       type);
190 			return 0;
191 		}
192 	}
193 
194 	return (insn >> shift) & mask;
195 }
196 
aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,u32 insn,u64 imm)197 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
198 				  u32 insn, u64 imm)
199 {
200 	u32 immlo, immhi, mask;
201 	int shift;
202 
203 	if (insn == AARCH64_BREAK_FAULT)
204 		return AARCH64_BREAK_FAULT;
205 
206 	switch (type) {
207 	case AARCH64_INSN_IMM_ADR:
208 		shift = 0;
209 		immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
210 		imm >>= ADR_IMM_HILOSPLIT;
211 		immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
212 		imm = immlo | immhi;
213 		mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
214 			(ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
215 		break;
216 	default:
217 		if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
218 			pr_err("%s: unknown immediate encoding %d\n", __func__,
219 			       type);
220 			return AARCH64_BREAK_FAULT;
221 		}
222 	}
223 
224 	/* Update the immediate field. */
225 	insn &= ~(mask << shift);
226 	insn |= (imm & mask) << shift;
227 
228 	return insn;
229 }
230 
aarch64_insn_decode_register(enum aarch64_insn_register_type type,u32 insn)231 u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
232 					u32 insn)
233 {
234 	int shift;
235 
236 	switch (type) {
237 	case AARCH64_INSN_REGTYPE_RT:
238 	case AARCH64_INSN_REGTYPE_RD:
239 		shift = 0;
240 		break;
241 	case AARCH64_INSN_REGTYPE_RN:
242 		shift = 5;
243 		break;
244 	case AARCH64_INSN_REGTYPE_RT2:
245 	case AARCH64_INSN_REGTYPE_RA:
246 		shift = 10;
247 		break;
248 	case AARCH64_INSN_REGTYPE_RM:
249 		shift = 16;
250 		break;
251 	default:
252 		pr_err("%s: unknown register type encoding %d\n", __func__,
253 		       type);
254 		return 0;
255 	}
256 
257 	return (insn >> shift) & GENMASK(4, 0);
258 }
259 
aarch64_insn_encode_register(enum aarch64_insn_register_type type,u32 insn,enum aarch64_insn_register reg)260 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
261 					u32 insn,
262 					enum aarch64_insn_register reg)
263 {
264 	int shift;
265 
266 	if (insn == AARCH64_BREAK_FAULT)
267 		return AARCH64_BREAK_FAULT;
268 
269 	if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
270 		pr_err("%s: unknown register encoding %d\n", __func__, reg);
271 		return AARCH64_BREAK_FAULT;
272 	}
273 
274 	switch (type) {
275 	case AARCH64_INSN_REGTYPE_RT:
276 	case AARCH64_INSN_REGTYPE_RD:
277 		shift = 0;
278 		break;
279 	case AARCH64_INSN_REGTYPE_RN:
280 		shift = 5;
281 		break;
282 	case AARCH64_INSN_REGTYPE_RT2:
283 	case AARCH64_INSN_REGTYPE_RA:
284 		shift = 10;
285 		break;
286 	case AARCH64_INSN_REGTYPE_RM:
287 	case AARCH64_INSN_REGTYPE_RS:
288 		shift = 16;
289 		break;
290 	default:
291 		pr_err("%s: unknown register type encoding %d\n", __func__,
292 		       type);
293 		return AARCH64_BREAK_FAULT;
294 	}
295 
296 	insn &= ~(GENMASK(4, 0) << shift);
297 	insn |= reg << shift;
298 
299 	return insn;
300 }
301 
302 static const u32 aarch64_insn_ldst_size[] = {
303 	[AARCH64_INSN_SIZE_8] = 0,
304 	[AARCH64_INSN_SIZE_16] = 1,
305 	[AARCH64_INSN_SIZE_32] = 2,
306 	[AARCH64_INSN_SIZE_64] = 3,
307 };
308 
aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,u32 insn)309 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
310 					 u32 insn)
311 {
312 	u32 size;
313 
314 	if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) {
315 		pr_err("%s: unknown size encoding %d\n", __func__, type);
316 		return AARCH64_BREAK_FAULT;
317 	}
318 
319 	size = aarch64_insn_ldst_size[type];
320 	insn &= ~GENMASK(31, 30);
321 	insn |= size << 30;
322 
323 	return insn;
324 }
325 
branch_imm_common(unsigned long pc,unsigned long addr,long range)326 static inline long branch_imm_common(unsigned long pc, unsigned long addr,
327 				     long range)
328 {
329 	long offset;
330 
331 	if ((pc & 0x3) || (addr & 0x3)) {
332 		pr_err("%s: A64 instructions must be word aligned\n", __func__);
333 		return range;
334 	}
335 
336 	offset = ((long)addr - (long)pc);
337 
338 	if (offset < -range || offset >= range) {
339 		pr_err("%s: offset out of range\n", __func__);
340 		return range;
341 	}
342 
343 	return offset;
344 }
345 
aarch64_insn_gen_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_branch_type type)346 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
347 					  enum aarch64_insn_branch_type type)
348 {
349 	u32 insn;
350 	long offset;
351 
352 	/*
353 	 * B/BL support [-128M, 128M) offset
354 	 * ARM64 virtual address arrangement guarantees all kernel and module
355 	 * texts are within +/-128M.
356 	 */
357 	offset = branch_imm_common(pc, addr, SZ_128M);
358 	if (offset >= SZ_128M)
359 		return AARCH64_BREAK_FAULT;
360 
361 	switch (type) {
362 	case AARCH64_INSN_BRANCH_LINK:
363 		insn = aarch64_insn_get_bl_value();
364 		break;
365 	case AARCH64_INSN_BRANCH_NOLINK:
366 		insn = aarch64_insn_get_b_value();
367 		break;
368 	default:
369 		pr_err("%s: unknown branch encoding %d\n", __func__, type);
370 		return AARCH64_BREAK_FAULT;
371 	}
372 
373 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
374 					     offset >> 2);
375 }
376 
aarch64_insn_gen_comp_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_register reg,enum aarch64_insn_variant variant,enum aarch64_insn_branch_type type)377 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
378 				     enum aarch64_insn_register reg,
379 				     enum aarch64_insn_variant variant,
380 				     enum aarch64_insn_branch_type type)
381 {
382 	u32 insn;
383 	long offset;
384 
385 	offset = branch_imm_common(pc, addr, SZ_1M);
386 	if (offset >= SZ_1M)
387 		return AARCH64_BREAK_FAULT;
388 
389 	switch (type) {
390 	case AARCH64_INSN_BRANCH_COMP_ZERO:
391 		insn = aarch64_insn_get_cbz_value();
392 		break;
393 	case AARCH64_INSN_BRANCH_COMP_NONZERO:
394 		insn = aarch64_insn_get_cbnz_value();
395 		break;
396 	default:
397 		pr_err("%s: unknown branch encoding %d\n", __func__, type);
398 		return AARCH64_BREAK_FAULT;
399 	}
400 
401 	switch (variant) {
402 	case AARCH64_INSN_VARIANT_32BIT:
403 		break;
404 	case AARCH64_INSN_VARIANT_64BIT:
405 		insn |= AARCH64_INSN_SF_BIT;
406 		break;
407 	default:
408 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
409 		return AARCH64_BREAK_FAULT;
410 	}
411 
412 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
413 
414 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
415 					     offset >> 2);
416 }
417 
aarch64_insn_gen_cond_branch_imm(unsigned long pc,unsigned long addr,enum aarch64_insn_condition cond)418 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
419 				     enum aarch64_insn_condition cond)
420 {
421 	u32 insn;
422 	long offset;
423 
424 	offset = branch_imm_common(pc, addr, SZ_1M);
425 
426 	insn = aarch64_insn_get_bcond_value();
427 
428 	if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
429 		pr_err("%s: unknown condition encoding %d\n", __func__, cond);
430 		return AARCH64_BREAK_FAULT;
431 	}
432 	insn |= cond;
433 
434 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
435 					     offset >> 2);
436 }
437 
aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)438 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_cr_op op)
439 {
440 	return aarch64_insn_get_hint_value() | op;
441 }
442 
aarch64_insn_gen_nop(void)443 u32 __kprobes aarch64_insn_gen_nop(void)
444 {
445 	return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
446 }
447 
aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,enum aarch64_insn_branch_type type)448 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
449 				enum aarch64_insn_branch_type type)
450 {
451 	u32 insn;
452 
453 	switch (type) {
454 	case AARCH64_INSN_BRANCH_NOLINK:
455 		insn = aarch64_insn_get_br_value();
456 		break;
457 	case AARCH64_INSN_BRANCH_LINK:
458 		insn = aarch64_insn_get_blr_value();
459 		break;
460 	case AARCH64_INSN_BRANCH_RETURN:
461 		insn = aarch64_insn_get_ret_value();
462 		break;
463 	default:
464 		pr_err("%s: unknown branch encoding %d\n", __func__, type);
465 		return AARCH64_BREAK_FAULT;
466 	}
467 
468 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
469 }
470 
aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,enum aarch64_insn_register base,enum aarch64_insn_register offset,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)471 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
472 				    enum aarch64_insn_register base,
473 				    enum aarch64_insn_register offset,
474 				    enum aarch64_insn_size_type size,
475 				    enum aarch64_insn_ldst_type type)
476 {
477 	u32 insn;
478 
479 	switch (type) {
480 	case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
481 		insn = aarch64_insn_get_ldr_reg_value();
482 		break;
483 	case AARCH64_INSN_LDST_STORE_REG_OFFSET:
484 		insn = aarch64_insn_get_str_reg_value();
485 		break;
486 	default:
487 		pr_err("%s: unknown load/store encoding %d\n", __func__, type);
488 		return AARCH64_BREAK_FAULT;
489 	}
490 
491 	insn = aarch64_insn_encode_ldst_size(size, insn);
492 
493 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
494 
495 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
496 					    base);
497 
498 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
499 					    offset);
500 }
501 
aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,enum aarch64_insn_register base,unsigned int imm,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)502 u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,
503 				    enum aarch64_insn_register base,
504 				    unsigned int imm,
505 				    enum aarch64_insn_size_type size,
506 				    enum aarch64_insn_ldst_type type)
507 {
508 	u32 insn;
509 	u32 shift;
510 
511 	if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) {
512 		pr_err("%s: unknown size encoding %d\n", __func__, type);
513 		return AARCH64_BREAK_FAULT;
514 	}
515 
516 	shift = aarch64_insn_ldst_size[size];
517 	if (imm & ~(BIT(12 + shift) - BIT(shift))) {
518 		pr_err("%s: invalid imm: %d\n", __func__, imm);
519 		return AARCH64_BREAK_FAULT;
520 	}
521 
522 	imm >>= shift;
523 
524 	switch (type) {
525 	case AARCH64_INSN_LDST_LOAD_IMM_OFFSET:
526 		insn = aarch64_insn_get_ldr_imm_value();
527 		break;
528 	case AARCH64_INSN_LDST_STORE_IMM_OFFSET:
529 		insn = aarch64_insn_get_str_imm_value();
530 		break;
531 	default:
532 		pr_err("%s: unknown load/store encoding %d\n", __func__, type);
533 		return AARCH64_BREAK_FAULT;
534 	}
535 
536 	insn = aarch64_insn_encode_ldst_size(size, insn);
537 
538 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
539 
540 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
541 					    base);
542 
543 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
544 }
545 
aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,enum aarch64_insn_register reg2,enum aarch64_insn_register base,int offset,enum aarch64_insn_variant variant,enum aarch64_insn_ldst_type type)546 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
547 				     enum aarch64_insn_register reg2,
548 				     enum aarch64_insn_register base,
549 				     int offset,
550 				     enum aarch64_insn_variant variant,
551 				     enum aarch64_insn_ldst_type type)
552 {
553 	u32 insn;
554 	int shift;
555 
556 	switch (type) {
557 	case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
558 		insn = aarch64_insn_get_ldp_pre_value();
559 		break;
560 	case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
561 		insn = aarch64_insn_get_stp_pre_value();
562 		break;
563 	case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
564 		insn = aarch64_insn_get_ldp_post_value();
565 		break;
566 	case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
567 		insn = aarch64_insn_get_stp_post_value();
568 		break;
569 	default:
570 		pr_err("%s: unknown load/store encoding %d\n", __func__, type);
571 		return AARCH64_BREAK_FAULT;
572 	}
573 
574 	switch (variant) {
575 	case AARCH64_INSN_VARIANT_32BIT:
576 		if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
577 			pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
578 			       __func__, offset);
579 			return AARCH64_BREAK_FAULT;
580 		}
581 		shift = 2;
582 		break;
583 	case AARCH64_INSN_VARIANT_64BIT:
584 		if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
585 			pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
586 			       __func__, offset);
587 			return AARCH64_BREAK_FAULT;
588 		}
589 		shift = 3;
590 		insn |= AARCH64_INSN_SF_BIT;
591 		break;
592 	default:
593 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
594 		return AARCH64_BREAK_FAULT;
595 	}
596 
597 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
598 					    reg1);
599 
600 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
601 					    reg2);
602 
603 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
604 					    base);
605 
606 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
607 					     offset >> shift);
608 }
609 
aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,enum aarch64_insn_register base,enum aarch64_insn_register state,enum aarch64_insn_size_type size,enum aarch64_insn_ldst_type type)610 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
611 				   enum aarch64_insn_register base,
612 				   enum aarch64_insn_register state,
613 				   enum aarch64_insn_size_type size,
614 				   enum aarch64_insn_ldst_type type)
615 {
616 	u32 insn;
617 
618 	switch (type) {
619 	case AARCH64_INSN_LDST_LOAD_EX:
620 	case AARCH64_INSN_LDST_LOAD_ACQ_EX:
621 		insn = aarch64_insn_get_load_ex_value();
622 		if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
623 			insn |= BIT(15);
624 		break;
625 	case AARCH64_INSN_LDST_STORE_EX:
626 	case AARCH64_INSN_LDST_STORE_REL_EX:
627 		insn = aarch64_insn_get_store_ex_value();
628 		if (type == AARCH64_INSN_LDST_STORE_REL_EX)
629 			insn |= BIT(15);
630 		break;
631 	default:
632 		pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
633 		return AARCH64_BREAK_FAULT;
634 	}
635 
636 	insn = aarch64_insn_encode_ldst_size(size, insn);
637 
638 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
639 					    reg);
640 
641 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
642 					    base);
643 
644 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
645 					    AARCH64_INSN_REG_ZR);
646 
647 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
648 					    state);
649 }
650 
651 #ifdef CONFIG_ARM64_LSE_ATOMICS
aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,u32 insn)652 static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
653 					  u32 insn)
654 {
655 	u32 order;
656 
657 	switch (type) {
658 	case AARCH64_INSN_MEM_ORDER_NONE:
659 		order = 0;
660 		break;
661 	case AARCH64_INSN_MEM_ORDER_ACQ:
662 		order = 2;
663 		break;
664 	case AARCH64_INSN_MEM_ORDER_REL:
665 		order = 1;
666 		break;
667 	case AARCH64_INSN_MEM_ORDER_ACQREL:
668 		order = 3;
669 		break;
670 	default:
671 		pr_err("%s: unknown mem order %d\n", __func__, type);
672 		return AARCH64_BREAK_FAULT;
673 	}
674 
675 	insn &= ~GENMASK(23, 22);
676 	insn |= order << 22;
677 
678 	return insn;
679 }
680 
aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,enum aarch64_insn_register address,enum aarch64_insn_register value,enum aarch64_insn_size_type size,enum aarch64_insn_mem_atomic_op op,enum aarch64_insn_mem_order_type order)681 u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
682 				  enum aarch64_insn_register address,
683 				  enum aarch64_insn_register value,
684 				  enum aarch64_insn_size_type size,
685 				  enum aarch64_insn_mem_atomic_op op,
686 				  enum aarch64_insn_mem_order_type order)
687 {
688 	u32 insn;
689 
690 	switch (op) {
691 	case AARCH64_INSN_MEM_ATOMIC_ADD:
692 		insn = aarch64_insn_get_ldadd_value();
693 		break;
694 	case AARCH64_INSN_MEM_ATOMIC_CLR:
695 		insn = aarch64_insn_get_ldclr_value();
696 		break;
697 	case AARCH64_INSN_MEM_ATOMIC_EOR:
698 		insn = aarch64_insn_get_ldeor_value();
699 		break;
700 	case AARCH64_INSN_MEM_ATOMIC_SET:
701 		insn = aarch64_insn_get_ldset_value();
702 		break;
703 	case AARCH64_INSN_MEM_ATOMIC_SWP:
704 		insn = aarch64_insn_get_swp_value();
705 		break;
706 	default:
707 		pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
708 		return AARCH64_BREAK_FAULT;
709 	}
710 
711 	switch (size) {
712 	case AARCH64_INSN_SIZE_32:
713 	case AARCH64_INSN_SIZE_64:
714 		break;
715 	default:
716 		pr_err("%s: unimplemented size encoding %d\n", __func__, size);
717 		return AARCH64_BREAK_FAULT;
718 	}
719 
720 	insn = aarch64_insn_encode_ldst_size(size, insn);
721 
722 	insn = aarch64_insn_encode_ldst_order(order, insn);
723 
724 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
725 					    result);
726 
727 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
728 					    address);
729 
730 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
731 					    value);
732 }
733 
aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,u32 insn)734 static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
735 					 u32 insn)
736 {
737 	u32 order;
738 
739 	switch (type) {
740 	case AARCH64_INSN_MEM_ORDER_NONE:
741 		order = 0;
742 		break;
743 	case AARCH64_INSN_MEM_ORDER_ACQ:
744 		order = BIT(22);
745 		break;
746 	case AARCH64_INSN_MEM_ORDER_REL:
747 		order = BIT(15);
748 		break;
749 	case AARCH64_INSN_MEM_ORDER_ACQREL:
750 		order = BIT(15) | BIT(22);
751 		break;
752 	default:
753 		pr_err("%s: unknown mem order %d\n", __func__, type);
754 		return AARCH64_BREAK_FAULT;
755 	}
756 
757 	insn &= ~(BIT(15) | BIT(22));
758 	insn |= order;
759 
760 	return insn;
761 }
762 
aarch64_insn_gen_cas(enum aarch64_insn_register result,enum aarch64_insn_register address,enum aarch64_insn_register value,enum aarch64_insn_size_type size,enum aarch64_insn_mem_order_type order)763 u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
764 			 enum aarch64_insn_register address,
765 			 enum aarch64_insn_register value,
766 			 enum aarch64_insn_size_type size,
767 			 enum aarch64_insn_mem_order_type order)
768 {
769 	u32 insn;
770 
771 	switch (size) {
772 	case AARCH64_INSN_SIZE_32:
773 	case AARCH64_INSN_SIZE_64:
774 		break;
775 	default:
776 		pr_err("%s: unimplemented size encoding %d\n", __func__, size);
777 		return AARCH64_BREAK_FAULT;
778 	}
779 
780 	insn = aarch64_insn_get_cas_value();
781 
782 	insn = aarch64_insn_encode_ldst_size(size, insn);
783 
784 	insn = aarch64_insn_encode_cas_order(order, insn);
785 
786 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
787 					    result);
788 
789 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
790 					    address);
791 
792 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
793 					    value);
794 }
795 #endif
796 
aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,enum aarch64_insn_prfm_target target,enum aarch64_insn_prfm_policy policy,u32 insn)797 static u32 aarch64_insn_encode_prfm_imm(enum aarch64_insn_prfm_type type,
798 					enum aarch64_insn_prfm_target target,
799 					enum aarch64_insn_prfm_policy policy,
800 					u32 insn)
801 {
802 	u32 imm_type = 0, imm_target = 0, imm_policy = 0;
803 
804 	switch (type) {
805 	case AARCH64_INSN_PRFM_TYPE_PLD:
806 		break;
807 	case AARCH64_INSN_PRFM_TYPE_PLI:
808 		imm_type = BIT(0);
809 		break;
810 	case AARCH64_INSN_PRFM_TYPE_PST:
811 		imm_type = BIT(1);
812 		break;
813 	default:
814 		pr_err("%s: unknown prfm type encoding %d\n", __func__, type);
815 		return AARCH64_BREAK_FAULT;
816 	}
817 
818 	switch (target) {
819 	case AARCH64_INSN_PRFM_TARGET_L1:
820 		break;
821 	case AARCH64_INSN_PRFM_TARGET_L2:
822 		imm_target = BIT(0);
823 		break;
824 	case AARCH64_INSN_PRFM_TARGET_L3:
825 		imm_target = BIT(1);
826 		break;
827 	default:
828 		pr_err("%s: unknown prfm target encoding %d\n", __func__, target);
829 		return AARCH64_BREAK_FAULT;
830 	}
831 
832 	switch (policy) {
833 	case AARCH64_INSN_PRFM_POLICY_KEEP:
834 		break;
835 	case AARCH64_INSN_PRFM_POLICY_STRM:
836 		imm_policy = BIT(0);
837 		break;
838 	default:
839 		pr_err("%s: unknown prfm policy encoding %d\n", __func__, policy);
840 		return AARCH64_BREAK_FAULT;
841 	}
842 
843 	/* In this case, imm5 is encoded into Rt field. */
844 	insn &= ~GENMASK(4, 0);
845 	insn |= imm_policy | (imm_target << 1) | (imm_type << 3);
846 
847 	return insn;
848 }
849 
aarch64_insn_gen_prefetch(enum aarch64_insn_register base,enum aarch64_insn_prfm_type type,enum aarch64_insn_prfm_target target,enum aarch64_insn_prfm_policy policy)850 u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base,
851 			      enum aarch64_insn_prfm_type type,
852 			      enum aarch64_insn_prfm_target target,
853 			      enum aarch64_insn_prfm_policy policy)
854 {
855 	u32 insn = aarch64_insn_get_prfm_value();
856 
857 	insn = aarch64_insn_encode_ldst_size(AARCH64_INSN_SIZE_64, insn);
858 
859 	insn = aarch64_insn_encode_prfm_imm(type, target, policy, insn);
860 
861 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
862 					    base);
863 
864 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, 0);
865 }
866 
aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,enum aarch64_insn_register src,int imm,enum aarch64_insn_variant variant,enum aarch64_insn_adsb_type type)867 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
868 				 enum aarch64_insn_register src,
869 				 int imm, enum aarch64_insn_variant variant,
870 				 enum aarch64_insn_adsb_type type)
871 {
872 	u32 insn;
873 
874 	switch (type) {
875 	case AARCH64_INSN_ADSB_ADD:
876 		insn = aarch64_insn_get_add_imm_value();
877 		break;
878 	case AARCH64_INSN_ADSB_SUB:
879 		insn = aarch64_insn_get_sub_imm_value();
880 		break;
881 	case AARCH64_INSN_ADSB_ADD_SETFLAGS:
882 		insn = aarch64_insn_get_adds_imm_value();
883 		break;
884 	case AARCH64_INSN_ADSB_SUB_SETFLAGS:
885 		insn = aarch64_insn_get_subs_imm_value();
886 		break;
887 	default:
888 		pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
889 		return AARCH64_BREAK_FAULT;
890 	}
891 
892 	switch (variant) {
893 	case AARCH64_INSN_VARIANT_32BIT:
894 		break;
895 	case AARCH64_INSN_VARIANT_64BIT:
896 		insn |= AARCH64_INSN_SF_BIT;
897 		break;
898 	default:
899 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
900 		return AARCH64_BREAK_FAULT;
901 	}
902 
903 	/* We can't encode more than a 24bit value (12bit + 12bit shift) */
904 	if (imm & ~(BIT(24) - 1))
905 		goto out;
906 
907 	/* If we have something in the top 12 bits... */
908 	if (imm & ~(SZ_4K - 1)) {
909 		/* ... and in the low 12 bits -> error */
910 		if (imm & (SZ_4K - 1))
911 			goto out;
912 
913 		imm >>= 12;
914 		insn |= AARCH64_INSN_LSL_12;
915 	}
916 
917 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
918 
919 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
920 
921 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
922 
923 out:
924 	pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
925 	return AARCH64_BREAK_FAULT;
926 }
927 
aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,enum aarch64_insn_register src,int immr,int imms,enum aarch64_insn_variant variant,enum aarch64_insn_bitfield_type type)928 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
929 			      enum aarch64_insn_register src,
930 			      int immr, int imms,
931 			      enum aarch64_insn_variant variant,
932 			      enum aarch64_insn_bitfield_type type)
933 {
934 	u32 insn;
935 	u32 mask;
936 
937 	switch (type) {
938 	case AARCH64_INSN_BITFIELD_MOVE:
939 		insn = aarch64_insn_get_bfm_value();
940 		break;
941 	case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
942 		insn = aarch64_insn_get_ubfm_value();
943 		break;
944 	case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
945 		insn = aarch64_insn_get_sbfm_value();
946 		break;
947 	default:
948 		pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
949 		return AARCH64_BREAK_FAULT;
950 	}
951 
952 	switch (variant) {
953 	case AARCH64_INSN_VARIANT_32BIT:
954 		mask = GENMASK(4, 0);
955 		break;
956 	case AARCH64_INSN_VARIANT_64BIT:
957 		insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
958 		mask = GENMASK(5, 0);
959 		break;
960 	default:
961 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
962 		return AARCH64_BREAK_FAULT;
963 	}
964 
965 	if (immr & ~mask) {
966 		pr_err("%s: invalid immr encoding %d\n", __func__, immr);
967 		return AARCH64_BREAK_FAULT;
968 	}
969 	if (imms & ~mask) {
970 		pr_err("%s: invalid imms encoding %d\n", __func__, imms);
971 		return AARCH64_BREAK_FAULT;
972 	}
973 
974 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
975 
976 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
977 
978 	insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
979 
980 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
981 }
982 
aarch64_insn_gen_movewide(enum aarch64_insn_register dst,int imm,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_movewide_type type)983 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
984 			      int imm, int shift,
985 			      enum aarch64_insn_variant variant,
986 			      enum aarch64_insn_movewide_type type)
987 {
988 	u32 insn;
989 
990 	switch (type) {
991 	case AARCH64_INSN_MOVEWIDE_ZERO:
992 		insn = aarch64_insn_get_movz_value();
993 		break;
994 	case AARCH64_INSN_MOVEWIDE_KEEP:
995 		insn = aarch64_insn_get_movk_value();
996 		break;
997 	case AARCH64_INSN_MOVEWIDE_INVERSE:
998 		insn = aarch64_insn_get_movn_value();
999 		break;
1000 	default:
1001 		pr_err("%s: unknown movewide encoding %d\n", __func__, type);
1002 		return AARCH64_BREAK_FAULT;
1003 	}
1004 
1005 	if (imm & ~(SZ_64K - 1)) {
1006 		pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
1007 		return AARCH64_BREAK_FAULT;
1008 	}
1009 
1010 	switch (variant) {
1011 	case AARCH64_INSN_VARIANT_32BIT:
1012 		if (shift != 0 && shift != 16) {
1013 			pr_err("%s: invalid shift encoding %d\n", __func__,
1014 			       shift);
1015 			return AARCH64_BREAK_FAULT;
1016 		}
1017 		break;
1018 	case AARCH64_INSN_VARIANT_64BIT:
1019 		insn |= AARCH64_INSN_SF_BIT;
1020 		if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
1021 			pr_err("%s: invalid shift encoding %d\n", __func__,
1022 			       shift);
1023 			return AARCH64_BREAK_FAULT;
1024 		}
1025 		break;
1026 	default:
1027 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1028 		return AARCH64_BREAK_FAULT;
1029 	}
1030 
1031 	insn |= (shift >> 4) << 21;
1032 
1033 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1034 
1035 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
1036 }
1037 
aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_adsb_type type)1038 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
1039 					 enum aarch64_insn_register src,
1040 					 enum aarch64_insn_register reg,
1041 					 int shift,
1042 					 enum aarch64_insn_variant variant,
1043 					 enum aarch64_insn_adsb_type type)
1044 {
1045 	u32 insn;
1046 
1047 	switch (type) {
1048 	case AARCH64_INSN_ADSB_ADD:
1049 		insn = aarch64_insn_get_add_value();
1050 		break;
1051 	case AARCH64_INSN_ADSB_SUB:
1052 		insn = aarch64_insn_get_sub_value();
1053 		break;
1054 	case AARCH64_INSN_ADSB_ADD_SETFLAGS:
1055 		insn = aarch64_insn_get_adds_value();
1056 		break;
1057 	case AARCH64_INSN_ADSB_SUB_SETFLAGS:
1058 		insn = aarch64_insn_get_subs_value();
1059 		break;
1060 	default:
1061 		pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
1062 		return AARCH64_BREAK_FAULT;
1063 	}
1064 
1065 	switch (variant) {
1066 	case AARCH64_INSN_VARIANT_32BIT:
1067 		if (shift & ~(SZ_32 - 1)) {
1068 			pr_err("%s: invalid shift encoding %d\n", __func__,
1069 			       shift);
1070 			return AARCH64_BREAK_FAULT;
1071 		}
1072 		break;
1073 	case AARCH64_INSN_VARIANT_64BIT:
1074 		insn |= AARCH64_INSN_SF_BIT;
1075 		if (shift & ~(SZ_64 - 1)) {
1076 			pr_err("%s: invalid shift encoding %d\n", __func__,
1077 			       shift);
1078 			return AARCH64_BREAK_FAULT;
1079 		}
1080 		break;
1081 	default:
1082 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1083 		return AARCH64_BREAK_FAULT;
1084 	}
1085 
1086 
1087 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1088 
1089 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1090 
1091 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1092 
1093 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1094 }
1095 
aarch64_insn_gen_data1(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_variant variant,enum aarch64_insn_data1_type type)1096 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
1097 			   enum aarch64_insn_register src,
1098 			   enum aarch64_insn_variant variant,
1099 			   enum aarch64_insn_data1_type type)
1100 {
1101 	u32 insn;
1102 
1103 	switch (type) {
1104 	case AARCH64_INSN_DATA1_REVERSE_16:
1105 		insn = aarch64_insn_get_rev16_value();
1106 		break;
1107 	case AARCH64_INSN_DATA1_REVERSE_32:
1108 		insn = aarch64_insn_get_rev32_value();
1109 		break;
1110 	case AARCH64_INSN_DATA1_REVERSE_64:
1111 		if (variant != AARCH64_INSN_VARIANT_64BIT) {
1112 			pr_err("%s: invalid variant for reverse64 %d\n",
1113 			       __func__, variant);
1114 			return AARCH64_BREAK_FAULT;
1115 		}
1116 		insn = aarch64_insn_get_rev64_value();
1117 		break;
1118 	default:
1119 		pr_err("%s: unknown data1 encoding %d\n", __func__, type);
1120 		return AARCH64_BREAK_FAULT;
1121 	}
1122 
1123 	switch (variant) {
1124 	case AARCH64_INSN_VARIANT_32BIT:
1125 		break;
1126 	case AARCH64_INSN_VARIANT_64BIT:
1127 		insn |= AARCH64_INSN_SF_BIT;
1128 		break;
1129 	default:
1130 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1131 		return AARCH64_BREAK_FAULT;
1132 	}
1133 
1134 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1135 
1136 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1137 }
1138 
aarch64_insn_gen_data2(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,enum aarch64_insn_variant variant,enum aarch64_insn_data2_type type)1139 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
1140 			   enum aarch64_insn_register src,
1141 			   enum aarch64_insn_register reg,
1142 			   enum aarch64_insn_variant variant,
1143 			   enum aarch64_insn_data2_type type)
1144 {
1145 	u32 insn;
1146 
1147 	switch (type) {
1148 	case AARCH64_INSN_DATA2_UDIV:
1149 		insn = aarch64_insn_get_udiv_value();
1150 		break;
1151 	case AARCH64_INSN_DATA2_SDIV:
1152 		insn = aarch64_insn_get_sdiv_value();
1153 		break;
1154 	case AARCH64_INSN_DATA2_LSLV:
1155 		insn = aarch64_insn_get_lslv_value();
1156 		break;
1157 	case AARCH64_INSN_DATA2_LSRV:
1158 		insn = aarch64_insn_get_lsrv_value();
1159 		break;
1160 	case AARCH64_INSN_DATA2_ASRV:
1161 		insn = aarch64_insn_get_asrv_value();
1162 		break;
1163 	case AARCH64_INSN_DATA2_RORV:
1164 		insn = aarch64_insn_get_rorv_value();
1165 		break;
1166 	default:
1167 		pr_err("%s: unknown data2 encoding %d\n", __func__, type);
1168 		return AARCH64_BREAK_FAULT;
1169 	}
1170 
1171 	switch (variant) {
1172 	case AARCH64_INSN_VARIANT_32BIT:
1173 		break;
1174 	case AARCH64_INSN_VARIANT_64BIT:
1175 		insn |= AARCH64_INSN_SF_BIT;
1176 		break;
1177 	default:
1178 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1179 		return AARCH64_BREAK_FAULT;
1180 	}
1181 
1182 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1183 
1184 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1185 
1186 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1187 }
1188 
aarch64_insn_gen_data3(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg1,enum aarch64_insn_register reg2,enum aarch64_insn_variant variant,enum aarch64_insn_data3_type type)1189 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
1190 			   enum aarch64_insn_register src,
1191 			   enum aarch64_insn_register reg1,
1192 			   enum aarch64_insn_register reg2,
1193 			   enum aarch64_insn_variant variant,
1194 			   enum aarch64_insn_data3_type type)
1195 {
1196 	u32 insn;
1197 
1198 	switch (type) {
1199 	case AARCH64_INSN_DATA3_MADD:
1200 		insn = aarch64_insn_get_madd_value();
1201 		break;
1202 	case AARCH64_INSN_DATA3_MSUB:
1203 		insn = aarch64_insn_get_msub_value();
1204 		break;
1205 	default:
1206 		pr_err("%s: unknown data3 encoding %d\n", __func__, type);
1207 		return AARCH64_BREAK_FAULT;
1208 	}
1209 
1210 	switch (variant) {
1211 	case AARCH64_INSN_VARIANT_32BIT:
1212 		break;
1213 	case AARCH64_INSN_VARIANT_64BIT:
1214 		insn |= AARCH64_INSN_SF_BIT;
1215 		break;
1216 	default:
1217 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1218 		return AARCH64_BREAK_FAULT;
1219 	}
1220 
1221 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1222 
1223 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
1224 
1225 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
1226 					    reg1);
1227 
1228 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
1229 					    reg2);
1230 }
1231 
aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_register reg,int shift,enum aarch64_insn_variant variant,enum aarch64_insn_logic_type type)1232 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
1233 					 enum aarch64_insn_register src,
1234 					 enum aarch64_insn_register reg,
1235 					 int shift,
1236 					 enum aarch64_insn_variant variant,
1237 					 enum aarch64_insn_logic_type type)
1238 {
1239 	u32 insn;
1240 
1241 	switch (type) {
1242 	case AARCH64_INSN_LOGIC_AND:
1243 		insn = aarch64_insn_get_and_value();
1244 		break;
1245 	case AARCH64_INSN_LOGIC_BIC:
1246 		insn = aarch64_insn_get_bic_value();
1247 		break;
1248 	case AARCH64_INSN_LOGIC_ORR:
1249 		insn = aarch64_insn_get_orr_value();
1250 		break;
1251 	case AARCH64_INSN_LOGIC_ORN:
1252 		insn = aarch64_insn_get_orn_value();
1253 		break;
1254 	case AARCH64_INSN_LOGIC_EOR:
1255 		insn = aarch64_insn_get_eor_value();
1256 		break;
1257 	case AARCH64_INSN_LOGIC_EON:
1258 		insn = aarch64_insn_get_eon_value();
1259 		break;
1260 	case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1261 		insn = aarch64_insn_get_ands_value();
1262 		break;
1263 	case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
1264 		insn = aarch64_insn_get_bics_value();
1265 		break;
1266 	default:
1267 		pr_err("%s: unknown logical encoding %d\n", __func__, type);
1268 		return AARCH64_BREAK_FAULT;
1269 	}
1270 
1271 	switch (variant) {
1272 	case AARCH64_INSN_VARIANT_32BIT:
1273 		if (shift & ~(SZ_32 - 1)) {
1274 			pr_err("%s: invalid shift encoding %d\n", __func__,
1275 			       shift);
1276 			return AARCH64_BREAK_FAULT;
1277 		}
1278 		break;
1279 	case AARCH64_INSN_VARIANT_64BIT:
1280 		insn |= AARCH64_INSN_SF_BIT;
1281 		if (shift & ~(SZ_64 - 1)) {
1282 			pr_err("%s: invalid shift encoding %d\n", __func__,
1283 			       shift);
1284 			return AARCH64_BREAK_FAULT;
1285 		}
1286 		break;
1287 	default:
1288 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1289 		return AARCH64_BREAK_FAULT;
1290 	}
1291 
1292 
1293 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1294 
1295 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1296 
1297 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1298 
1299 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1300 }
1301 
1302 /*
1303  * MOV (register) is architecturally an alias of ORR (shifted register) where
1304  * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
1305  */
aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,enum aarch64_insn_register src,enum aarch64_insn_variant variant)1306 u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
1307 			      enum aarch64_insn_register src,
1308 			      enum aarch64_insn_variant variant)
1309 {
1310 	return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
1311 						    src, 0, variant,
1312 						    AARCH64_INSN_LOGIC_ORR);
1313 }
1314 
aarch64_insn_gen_adr(unsigned long pc,unsigned long addr,enum aarch64_insn_register reg,enum aarch64_insn_adr_type type)1315 u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
1316 			 enum aarch64_insn_register reg,
1317 			 enum aarch64_insn_adr_type type)
1318 {
1319 	u32 insn;
1320 	s32 offset;
1321 
1322 	switch (type) {
1323 	case AARCH64_INSN_ADR_TYPE_ADR:
1324 		insn = aarch64_insn_get_adr_value();
1325 		offset = addr - pc;
1326 		break;
1327 	case AARCH64_INSN_ADR_TYPE_ADRP:
1328 		insn = aarch64_insn_get_adrp_value();
1329 		offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
1330 		break;
1331 	default:
1332 		pr_err("%s: unknown adr encoding %d\n", __func__, type);
1333 		return AARCH64_BREAK_FAULT;
1334 	}
1335 
1336 	if (offset < -SZ_1M || offset >= SZ_1M)
1337 		return AARCH64_BREAK_FAULT;
1338 
1339 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
1340 
1341 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
1342 }
1343 
1344 /*
1345  * Decode the imm field of a branch, and return the byte offset as a
1346  * signed value (so it can be used when computing a new branch
1347  * target).
1348  */
aarch64_get_branch_offset(u32 insn)1349 s32 aarch64_get_branch_offset(u32 insn)
1350 {
1351 	s32 imm;
1352 
1353 	if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
1354 		imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
1355 		return (imm << 6) >> 4;
1356 	}
1357 
1358 	if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1359 	    aarch64_insn_is_bcond(insn)) {
1360 		imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
1361 		return (imm << 13) >> 11;
1362 	}
1363 
1364 	if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
1365 		imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
1366 		return (imm << 18) >> 16;
1367 	}
1368 
1369 	/* Unhandled instruction */
1370 	BUG();
1371 }
1372 
1373 /*
1374  * Encode the displacement of a branch in the imm field and return the
1375  * updated instruction.
1376  */
aarch64_set_branch_offset(u32 insn,s32 offset)1377 u32 aarch64_set_branch_offset(u32 insn, s32 offset)
1378 {
1379 	if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
1380 		return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
1381 						     offset >> 2);
1382 
1383 	if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1384 	    aarch64_insn_is_bcond(insn))
1385 		return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
1386 						     offset >> 2);
1387 
1388 	if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
1389 		return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
1390 						     offset >> 2);
1391 
1392 	/* Unhandled instruction */
1393 	BUG();
1394 }
1395 
aarch64_insn_adrp_get_offset(u32 insn)1396 s32 aarch64_insn_adrp_get_offset(u32 insn)
1397 {
1398 	BUG_ON(!aarch64_insn_is_adrp(insn));
1399 	return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12;
1400 }
1401 
aarch64_insn_adrp_set_offset(u32 insn,s32 offset)1402 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset)
1403 {
1404 	BUG_ON(!aarch64_insn_is_adrp(insn));
1405 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn,
1406 						offset >> 12);
1407 }
1408 
1409 /*
1410  * Extract the Op/CR data from a msr/mrs instruction.
1411  */
aarch64_insn_extract_system_reg(u32 insn)1412 u32 aarch64_insn_extract_system_reg(u32 insn)
1413 {
1414 	return (insn & 0x1FFFE0) >> 5;
1415 }
1416 
aarch32_insn_is_wide(u32 insn)1417 bool aarch32_insn_is_wide(u32 insn)
1418 {
1419 	return insn >= 0xe800;
1420 }
1421 
1422 /*
1423  * Macros/defines for extracting register numbers from instruction.
1424  */
aarch32_insn_extract_reg_num(u32 insn,int offset)1425 u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
1426 {
1427 	return (insn & (0xf << offset)) >> offset;
1428 }
1429 
1430 #define OPC2_MASK	0x7
1431 #define OPC2_OFFSET	5
aarch32_insn_mcr_extract_opc2(u32 insn)1432 u32 aarch32_insn_mcr_extract_opc2(u32 insn)
1433 {
1434 	return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
1435 }
1436 
1437 #define CRM_MASK	0xf
aarch32_insn_mcr_extract_crm(u32 insn)1438 u32 aarch32_insn_mcr_extract_crm(u32 insn)
1439 {
1440 	return insn & CRM_MASK;
1441 }
1442 
range_of_ones(u64 val)1443 static bool range_of_ones(u64 val)
1444 {
1445 	/* Doesn't handle full ones or full zeroes */
1446 	u64 sval = val >> __ffs64(val);
1447 
1448 	/* One of Sean Eron Anderson's bithack tricks */
1449 	return ((sval + 1) & (sval)) == 0;
1450 }
1451 
aarch64_encode_immediate(u64 imm,enum aarch64_insn_variant variant,u32 insn)1452 static u32 aarch64_encode_immediate(u64 imm,
1453 				    enum aarch64_insn_variant variant,
1454 				    u32 insn)
1455 {
1456 	unsigned int immr, imms, n, ones, ror, esz, tmp;
1457 	u64 mask;
1458 
1459 	switch (variant) {
1460 	case AARCH64_INSN_VARIANT_32BIT:
1461 		esz = 32;
1462 		break;
1463 	case AARCH64_INSN_VARIANT_64BIT:
1464 		insn |= AARCH64_INSN_SF_BIT;
1465 		esz = 64;
1466 		break;
1467 	default:
1468 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1469 		return AARCH64_BREAK_FAULT;
1470 	}
1471 
1472 	mask = GENMASK(esz - 1, 0);
1473 
1474 	/* Can't encode full zeroes, full ones, or value wider than the mask */
1475 	if (!imm || imm == mask || imm & ~mask)
1476 		return AARCH64_BREAK_FAULT;
1477 
1478 	/*
1479 	 * Inverse of Replicate(). Try to spot a repeating pattern
1480 	 * with a pow2 stride.
1481 	 */
1482 	for (tmp = esz / 2; tmp >= 2; tmp /= 2) {
1483 		u64 emask = BIT(tmp) - 1;
1484 
1485 		if ((imm & emask) != ((imm >> tmp) & emask))
1486 			break;
1487 
1488 		esz = tmp;
1489 		mask = emask;
1490 	}
1491 
1492 	/* N is only set if we're encoding a 64bit value */
1493 	n = esz == 64;
1494 
1495 	/* Trim imm to the element size */
1496 	imm &= mask;
1497 
1498 	/* That's how many ones we need to encode */
1499 	ones = hweight64(imm);
1500 
1501 	/*
1502 	 * imms is set to (ones - 1), prefixed with a string of ones
1503 	 * and a zero if they fit. Cap it to 6 bits.
1504 	 */
1505 	imms  = ones - 1;
1506 	imms |= 0xf << ffs(esz);
1507 	imms &= BIT(6) - 1;
1508 
1509 	/* Compute the rotation */
1510 	if (range_of_ones(imm)) {
1511 		/*
1512 		 * Pattern: 0..01..10..0
1513 		 *
1514 		 * Compute how many rotate we need to align it right
1515 		 */
1516 		ror = __ffs64(imm);
1517 	} else {
1518 		/*
1519 		 * Pattern: 0..01..10..01..1
1520 		 *
1521 		 * Fill the unused top bits with ones, and check if
1522 		 * the result is a valid immediate (all ones with a
1523 		 * contiguous ranges of zeroes).
1524 		 */
1525 		imm |= ~mask;
1526 		if (!range_of_ones(~imm))
1527 			return AARCH64_BREAK_FAULT;
1528 
1529 		/*
1530 		 * Compute the rotation to get a continuous set of
1531 		 * ones, with the first bit set at position 0
1532 		 */
1533 		ror = fls64(~imm);
1534 	}
1535 
1536 	/*
1537 	 * immr is the number of bits we need to rotate back to the
1538 	 * original set of ones. Note that this is relative to the
1539 	 * element size...
1540 	 */
1541 	immr = (esz - ror) % esz;
1542 
1543 	insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n);
1544 	insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
1545 	return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
1546 }
1547 
aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,enum aarch64_insn_variant variant,enum aarch64_insn_register Rn,enum aarch64_insn_register Rd,u64 imm)1548 u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
1549 				       enum aarch64_insn_variant variant,
1550 				       enum aarch64_insn_register Rn,
1551 				       enum aarch64_insn_register Rd,
1552 				       u64 imm)
1553 {
1554 	u32 insn;
1555 
1556 	switch (type) {
1557 	case AARCH64_INSN_LOGIC_AND:
1558 		insn = aarch64_insn_get_and_imm_value();
1559 		break;
1560 	case AARCH64_INSN_LOGIC_ORR:
1561 		insn = aarch64_insn_get_orr_imm_value();
1562 		break;
1563 	case AARCH64_INSN_LOGIC_EOR:
1564 		insn = aarch64_insn_get_eor_imm_value();
1565 		break;
1566 	case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1567 		insn = aarch64_insn_get_ands_imm_value();
1568 		break;
1569 	default:
1570 		pr_err("%s: unknown logical encoding %d\n", __func__, type);
1571 		return AARCH64_BREAK_FAULT;
1572 	}
1573 
1574 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1575 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1576 	return aarch64_encode_immediate(imm, variant, insn);
1577 }
1578 
aarch64_insn_gen_extr(enum aarch64_insn_variant variant,enum aarch64_insn_register Rm,enum aarch64_insn_register Rn,enum aarch64_insn_register Rd,u8 lsb)1579 u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
1580 			  enum aarch64_insn_register Rm,
1581 			  enum aarch64_insn_register Rn,
1582 			  enum aarch64_insn_register Rd,
1583 			  u8 lsb)
1584 {
1585 	u32 insn;
1586 
1587 	insn = aarch64_insn_get_extr_value();
1588 
1589 	switch (variant) {
1590 	case AARCH64_INSN_VARIANT_32BIT:
1591 		if (lsb > 31)
1592 			return AARCH64_BREAK_FAULT;
1593 		break;
1594 	case AARCH64_INSN_VARIANT_64BIT:
1595 		if (lsb > 63)
1596 			return AARCH64_BREAK_FAULT;
1597 		insn |= AARCH64_INSN_SF_BIT;
1598 		insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1);
1599 		break;
1600 	default:
1601 		pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1602 		return AARCH64_BREAK_FAULT;
1603 	}
1604 
1605 	insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb);
1606 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1607 	insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1608 	return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
1609 }
1610 
aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)1611 u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1612 {
1613 	u32 opt;
1614 	u32 insn;
1615 
1616 	switch (type) {
1617 	case AARCH64_INSN_MB_SY:
1618 		opt = 0xf;
1619 		break;
1620 	case AARCH64_INSN_MB_ST:
1621 		opt = 0xe;
1622 		break;
1623 	case AARCH64_INSN_MB_LD:
1624 		opt = 0xd;
1625 		break;
1626 	case AARCH64_INSN_MB_ISH:
1627 		opt = 0xb;
1628 		break;
1629 	case AARCH64_INSN_MB_ISHST:
1630 		opt = 0xa;
1631 		break;
1632 	case AARCH64_INSN_MB_ISHLD:
1633 		opt = 0x9;
1634 		break;
1635 	case AARCH64_INSN_MB_NSH:
1636 		opt = 0x7;
1637 		break;
1638 	case AARCH64_INSN_MB_NSHST:
1639 		opt = 0x6;
1640 		break;
1641 	case AARCH64_INSN_MB_NSHLD:
1642 		opt = 0x5;
1643 		break;
1644 	default:
1645 		pr_err("%s: unknown dmb type %d\n", __func__, type);
1646 		return AARCH64_BREAK_FAULT;
1647 	}
1648 
1649 	insn = aarch64_insn_get_dmb_value();
1650 	insn &= ~GENMASK(11, 8);
1651 	insn |= (opt << 8);
1652 
1653 	return insn;
1654 }
1655