1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * Common functionality for RV32 and RV64 BPF JIT compilers
4  *
5  * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6  *
7  */
8 
9 #ifndef _BPF_JIT_H
10 #define _BPF_JIT_H
11 
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <asm/cacheflush.h>
15 
rvc_enabled(void)16 static inline bool rvc_enabled(void)
17 {
18 	return IS_ENABLED(CONFIG_RISCV_ISA_C);
19 }
20 
21 enum {
22 	RV_REG_ZERO =	0,	/* The constant value 0 */
23 	RV_REG_RA =	1,	/* Return address */
24 	RV_REG_SP =	2,	/* Stack pointer */
25 	RV_REG_GP =	3,	/* Global pointer */
26 	RV_REG_TP =	4,	/* Thread pointer */
27 	RV_REG_T0 =	5,	/* Temporaries */
28 	RV_REG_T1 =	6,
29 	RV_REG_T2 =	7,
30 	RV_REG_FP =	8,	/* Saved register/frame pointer */
31 	RV_REG_S1 =	9,	/* Saved register */
32 	RV_REG_A0 =	10,	/* Function argument/return values */
33 	RV_REG_A1 =	11,	/* Function arguments */
34 	RV_REG_A2 =	12,
35 	RV_REG_A3 =	13,
36 	RV_REG_A4 =	14,
37 	RV_REG_A5 =	15,
38 	RV_REG_A6 =	16,
39 	RV_REG_A7 =	17,
40 	RV_REG_S2 =	18,	/* Saved registers */
41 	RV_REG_S3 =	19,
42 	RV_REG_S4 =	20,
43 	RV_REG_S5 =	21,
44 	RV_REG_S6 =	22,
45 	RV_REG_S7 =	23,
46 	RV_REG_S8 =	24,
47 	RV_REG_S9 =	25,
48 	RV_REG_S10 =	26,
49 	RV_REG_S11 =	27,
50 	RV_REG_T3 =	28,	/* Temporaries */
51 	RV_REG_T4 =	29,
52 	RV_REG_T5 =	30,
53 	RV_REG_T6 =	31,
54 };
55 
is_creg(u8 reg)56 static inline bool is_creg(u8 reg)
57 {
58 	return (1 << reg) & (BIT(RV_REG_FP) |
59 			     BIT(RV_REG_S1) |
60 			     BIT(RV_REG_A0) |
61 			     BIT(RV_REG_A1) |
62 			     BIT(RV_REG_A2) |
63 			     BIT(RV_REG_A3) |
64 			     BIT(RV_REG_A4) |
65 			     BIT(RV_REG_A5));
66 }
67 
68 struct rv_jit_context {
69 	struct bpf_prog *prog;
70 	u16 *insns;		/* RV insns */
71 	u16 *ro_insns;
72 	int ninsns;
73 	int prologue_len;
74 	int epilogue_offset;
75 	int *offset;		/* BPF to RV */
76 	int nexentries;
77 	unsigned long flags;
78 	int stack_size;
79 };
80 
81 /* Convert from ninsns to bytes. */
ninsns_rvoff(int ninsns)82 static inline int ninsns_rvoff(int ninsns)
83 {
84 	return ninsns << 1;
85 }
86 
87 struct rv_jit_data {
88 	struct bpf_binary_header *header;
89 	struct bpf_binary_header *ro_header;
90 	u8 *image;
91 	u8 *ro_image;
92 	struct rv_jit_context ctx;
93 };
94 
bpf_fill_ill_insns(void * area,unsigned int size)95 static inline void bpf_fill_ill_insns(void *area, unsigned int size)
96 {
97 	memset(area, 0, size);
98 }
99 
bpf_flush_icache(void * start,void * end)100 static inline void bpf_flush_icache(void *start, void *end)
101 {
102 	flush_icache_range((unsigned long)start, (unsigned long)end);
103 }
104 
105 /* Emit a 4-byte riscv instruction. */
emit(const u32 insn,struct rv_jit_context * ctx)106 static inline void emit(const u32 insn, struct rv_jit_context *ctx)
107 {
108 	if (ctx->insns) {
109 		ctx->insns[ctx->ninsns] = insn;
110 		ctx->insns[ctx->ninsns + 1] = (insn >> 16);
111 	}
112 
113 	ctx->ninsns += 2;
114 }
115 
116 /* Emit a 2-byte riscv compressed instruction. */
emitc(const u16 insn,struct rv_jit_context * ctx)117 static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
118 {
119 	BUILD_BUG_ON(!rvc_enabled());
120 
121 	if (ctx->insns)
122 		ctx->insns[ctx->ninsns] = insn;
123 
124 	ctx->ninsns++;
125 }
126 
epilogue_offset(struct rv_jit_context * ctx)127 static inline int epilogue_offset(struct rv_jit_context *ctx)
128 {
129 	int to = ctx->epilogue_offset, from = ctx->ninsns;
130 
131 	return ninsns_rvoff(to - from);
132 }
133 
134 /* Return -1 or inverted cond. */
invert_bpf_cond(u8 cond)135 static inline int invert_bpf_cond(u8 cond)
136 {
137 	switch (cond) {
138 	case BPF_JEQ:
139 		return BPF_JNE;
140 	case BPF_JGT:
141 		return BPF_JLE;
142 	case BPF_JLT:
143 		return BPF_JGE;
144 	case BPF_JGE:
145 		return BPF_JLT;
146 	case BPF_JLE:
147 		return BPF_JGT;
148 	case BPF_JNE:
149 		return BPF_JEQ;
150 	case BPF_JSGT:
151 		return BPF_JSLE;
152 	case BPF_JSLT:
153 		return BPF_JSGE;
154 	case BPF_JSGE:
155 		return BPF_JSLT;
156 	case BPF_JSLE:
157 		return BPF_JSGT;
158 	}
159 	return -1;
160 }
161 
is_6b_int(long val)162 static inline bool is_6b_int(long val)
163 {
164 	return -(1L << 5) <= val && val < (1L << 5);
165 }
166 
is_7b_uint(unsigned long val)167 static inline bool is_7b_uint(unsigned long val)
168 {
169 	return val < (1UL << 7);
170 }
171 
is_8b_uint(unsigned long val)172 static inline bool is_8b_uint(unsigned long val)
173 {
174 	return val < (1UL << 8);
175 }
176 
is_9b_uint(unsigned long val)177 static inline bool is_9b_uint(unsigned long val)
178 {
179 	return val < (1UL << 9);
180 }
181 
is_10b_int(long val)182 static inline bool is_10b_int(long val)
183 {
184 	return -(1L << 9) <= val && val < (1L << 9);
185 }
186 
is_10b_uint(unsigned long val)187 static inline bool is_10b_uint(unsigned long val)
188 {
189 	return val < (1UL << 10);
190 }
191 
is_12b_int(long val)192 static inline bool is_12b_int(long val)
193 {
194 	return -(1L << 11) <= val && val < (1L << 11);
195 }
196 
is_12b_check(int off,int insn)197 static inline int is_12b_check(int off, int insn)
198 {
199 	if (!is_12b_int(off)) {
200 		pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
201 		       insn, (int)off);
202 		return -1;
203 	}
204 	return 0;
205 }
206 
is_13b_int(long val)207 static inline bool is_13b_int(long val)
208 {
209 	return -(1L << 12) <= val && val < (1L << 12);
210 }
211 
is_21b_int(long val)212 static inline bool is_21b_int(long val)
213 {
214 	return -(1L << 20) <= val && val < (1L << 20);
215 }
216 
rv_offset(int insn,int off,struct rv_jit_context * ctx)217 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
218 {
219 	int from, to;
220 
221 	off++; /* BPF branch is from PC+1, RV is from PC */
222 	from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len;
223 	to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len;
224 	return ninsns_rvoff(to - from);
225 }
226 
227 /* Instruction formats. */
228 
rv_r_insn(u8 funct7,u8 rs2,u8 rs1,u8 funct3,u8 rd,u8 opcode)229 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
230 			    u8 opcode)
231 {
232 	return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
233 		(rd << 7) | opcode;
234 }
235 
rv_i_insn(u16 imm11_0,u8 rs1,u8 funct3,u8 rd,u8 opcode)236 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
237 {
238 	return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
239 		opcode;
240 }
241 
rv_s_insn(u16 imm11_0,u8 rs2,u8 rs1,u8 funct3,u8 opcode)242 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
243 {
244 	u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
245 
246 	return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
247 		(imm4_0 << 7) | opcode;
248 }
249 
rv_b_insn(u16 imm12_1,u8 rs2,u8 rs1,u8 funct3,u8 opcode)250 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
251 {
252 	u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
253 	u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
254 
255 	return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
256 		(imm4_1 << 7) | opcode;
257 }
258 
rv_u_insn(u32 imm31_12,u8 rd,u8 opcode)259 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
260 {
261 	return (imm31_12 << 12) | (rd << 7) | opcode;
262 }
263 
rv_j_insn(u32 imm20_1,u8 rd,u8 opcode)264 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
265 {
266 	u32 imm;
267 
268 	imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
269 		((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
270 
271 	return (imm << 12) | (rd << 7) | opcode;
272 }
273 
rv_amo_insn(u8 funct5,u8 aq,u8 rl,u8 rs2,u8 rs1,u8 funct3,u8 rd,u8 opcode)274 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
275 			      u8 funct3, u8 rd, u8 opcode)
276 {
277 	u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
278 
279 	return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
280 }
281 
282 /* RISC-V compressed instruction formats. */
283 
rv_cr_insn(u8 funct4,u8 rd,u8 rs2,u8 op)284 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
285 {
286 	return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
287 }
288 
rv_ci_insn(u8 funct3,u32 imm6,u8 rd,u8 op)289 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
290 {
291 	u32 imm;
292 
293 	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
294 	return (funct3 << 13) | (rd << 7) | op | imm;
295 }
296 
rv_css_insn(u8 funct3,u32 uimm,u8 rs2,u8 op)297 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
298 {
299 	return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
300 }
301 
rv_ciw_insn(u8 funct3,u32 uimm,u8 rd,u8 op)302 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
303 {
304 	return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
305 }
306 
rv_cl_insn(u8 funct3,u32 imm_hi,u8 rs1,u32 imm_lo,u8 rd,u8 op)307 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
308 			     u8 op)
309 {
310 	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
311 		(imm_lo << 5) | ((rd & 0x7) << 2) | op;
312 }
313 
rv_cs_insn(u8 funct3,u32 imm_hi,u8 rs1,u32 imm_lo,u8 rs2,u8 op)314 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
315 			     u8 op)
316 {
317 	return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
318 		(imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
319 }
320 
rv_ca_insn(u8 funct6,u8 rd,u8 funct2,u8 rs2,u8 op)321 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
322 {
323 	return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
324 		((rs2 & 0x7) << 2) | op;
325 }
326 
rv_cb_insn(u8 funct3,u32 imm6,u8 funct2,u8 rd,u8 op)327 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
328 {
329 	u32 imm;
330 
331 	imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
332 	return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
333 }
334 
335 /* Instructions shared by both RV32 and RV64. */
336 
rv_addi(u8 rd,u8 rs1,u16 imm11_0)337 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
338 {
339 	return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
340 }
341 
rv_andi(u8 rd,u8 rs1,u16 imm11_0)342 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
343 {
344 	return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
345 }
346 
rv_ori(u8 rd,u8 rs1,u16 imm11_0)347 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
348 {
349 	return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
350 }
351 
rv_xori(u8 rd,u8 rs1,u16 imm11_0)352 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
353 {
354 	return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
355 }
356 
rv_slli(u8 rd,u8 rs1,u16 imm11_0)357 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
358 {
359 	return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
360 }
361 
rv_srli(u8 rd,u8 rs1,u16 imm11_0)362 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
363 {
364 	return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
365 }
366 
rv_srai(u8 rd,u8 rs1,u16 imm11_0)367 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
368 {
369 	return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
370 }
371 
rv_lui(u8 rd,u32 imm31_12)372 static inline u32 rv_lui(u8 rd, u32 imm31_12)
373 {
374 	return rv_u_insn(imm31_12, rd, 0x37);
375 }
376 
rv_auipc(u8 rd,u32 imm31_12)377 static inline u32 rv_auipc(u8 rd, u32 imm31_12)
378 {
379 	return rv_u_insn(imm31_12, rd, 0x17);
380 }
381 
rv_add(u8 rd,u8 rs1,u8 rs2)382 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
383 {
384 	return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
385 }
386 
rv_sub(u8 rd,u8 rs1,u8 rs2)387 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
388 {
389 	return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
390 }
391 
rv_sltu(u8 rd,u8 rs1,u8 rs2)392 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
393 {
394 	return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
395 }
396 
rv_and(u8 rd,u8 rs1,u8 rs2)397 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
398 {
399 	return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
400 }
401 
rv_or(u8 rd,u8 rs1,u8 rs2)402 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
403 {
404 	return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
405 }
406 
rv_xor(u8 rd,u8 rs1,u8 rs2)407 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
408 {
409 	return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
410 }
411 
rv_sll(u8 rd,u8 rs1,u8 rs2)412 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
413 {
414 	return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
415 }
416 
rv_srl(u8 rd,u8 rs1,u8 rs2)417 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
418 {
419 	return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
420 }
421 
rv_sra(u8 rd,u8 rs1,u8 rs2)422 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
423 {
424 	return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
425 }
426 
rv_mul(u8 rd,u8 rs1,u8 rs2)427 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
428 {
429 	return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
430 }
431 
rv_mulhu(u8 rd,u8 rs1,u8 rs2)432 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
433 {
434 	return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
435 }
436 
rv_div(u8 rd,u8 rs1,u8 rs2)437 static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2)
438 {
439 	return rv_r_insn(1, rs2, rs1, 4, rd, 0x33);
440 }
441 
rv_divu(u8 rd,u8 rs1,u8 rs2)442 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
443 {
444 	return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
445 }
446 
rv_rem(u8 rd,u8 rs1,u8 rs2)447 static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2)
448 {
449 	return rv_r_insn(1, rs2, rs1, 6, rd, 0x33);
450 }
451 
rv_remu(u8 rd,u8 rs1,u8 rs2)452 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
453 {
454 	return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
455 }
456 
rv_jal(u8 rd,u32 imm20_1)457 static inline u32 rv_jal(u8 rd, u32 imm20_1)
458 {
459 	return rv_j_insn(imm20_1, rd, 0x6f);
460 }
461 
rv_jalr(u8 rd,u8 rs1,u16 imm11_0)462 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
463 {
464 	return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
465 }
466 
rv_beq(u8 rs1,u8 rs2,u16 imm12_1)467 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
468 {
469 	return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
470 }
471 
rv_bne(u8 rs1,u8 rs2,u16 imm12_1)472 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
473 {
474 	return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
475 }
476 
rv_bltu(u8 rs1,u8 rs2,u16 imm12_1)477 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
478 {
479 	return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
480 }
481 
rv_bgtu(u8 rs1,u8 rs2,u16 imm12_1)482 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
483 {
484 	return rv_bltu(rs2, rs1, imm12_1);
485 }
486 
rv_bgeu(u8 rs1,u8 rs2,u16 imm12_1)487 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
488 {
489 	return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
490 }
491 
rv_bleu(u8 rs1,u8 rs2,u16 imm12_1)492 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
493 {
494 	return rv_bgeu(rs2, rs1, imm12_1);
495 }
496 
rv_blt(u8 rs1,u8 rs2,u16 imm12_1)497 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
498 {
499 	return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
500 }
501 
rv_bgt(u8 rs1,u8 rs2,u16 imm12_1)502 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
503 {
504 	return rv_blt(rs2, rs1, imm12_1);
505 }
506 
rv_bge(u8 rs1,u8 rs2,u16 imm12_1)507 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
508 {
509 	return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
510 }
511 
rv_ble(u8 rs1,u8 rs2,u16 imm12_1)512 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
513 {
514 	return rv_bge(rs2, rs1, imm12_1);
515 }
516 
rv_lb(u8 rd,u16 imm11_0,u8 rs1)517 static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1)
518 {
519 	return rv_i_insn(imm11_0, rs1, 0, rd, 0x03);
520 }
521 
rv_lh(u8 rd,u16 imm11_0,u8 rs1)522 static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1)
523 {
524 	return rv_i_insn(imm11_0, rs1, 1, rd, 0x03);
525 }
526 
rv_lw(u8 rd,u16 imm11_0,u8 rs1)527 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
528 {
529 	return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
530 }
531 
rv_lbu(u8 rd,u16 imm11_0,u8 rs1)532 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
533 {
534 	return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
535 }
536 
rv_lhu(u8 rd,u16 imm11_0,u8 rs1)537 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
538 {
539 	return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
540 }
541 
rv_sb(u8 rs1,u16 imm11_0,u8 rs2)542 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
543 {
544 	return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
545 }
546 
rv_sh(u8 rs1,u16 imm11_0,u8 rs2)547 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
548 {
549 	return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
550 }
551 
rv_sw(u8 rs1,u16 imm11_0,u8 rs2)552 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
553 {
554 	return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
555 }
556 
rv_amoadd_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)557 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
558 {
559 	return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
560 }
561 
rv_amoand_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)562 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
563 {
564 	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
565 }
566 
rv_amoor_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)567 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
568 {
569 	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
570 }
571 
rv_amoxor_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)572 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
573 {
574 	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
575 }
576 
rv_amoswap_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)577 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
578 {
579 	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
580 }
581 
rv_lr_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)582 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
583 {
584 	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
585 }
586 
rv_sc_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)587 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
588 {
589 	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
590 }
591 
rv_fence(u8 pred,u8 succ)592 static inline u32 rv_fence(u8 pred, u8 succ)
593 {
594 	u16 imm11_0 = pred << 4 | succ;
595 
596 	return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
597 }
598 
rv_nop(void)599 static inline u32 rv_nop(void)
600 {
601 	return rv_i_insn(0, 0, 0, 0, 0x13);
602 }
603 
604 /* RVC instrutions. */
605 
rvc_addi4spn(u8 rd,u32 imm10)606 static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
607 {
608 	u32 imm;
609 
610 	imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
611 		((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
612 	return rv_ciw_insn(0x0, imm, rd, 0x0);
613 }
614 
rvc_lw(u8 rd,u32 imm7,u8 rs1)615 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
616 {
617 	u32 imm_hi, imm_lo;
618 
619 	imm_hi = (imm7 & 0x38) >> 3;
620 	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
621 	return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
622 }
623 
rvc_sw(u8 rs1,u32 imm7,u8 rs2)624 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
625 {
626 	u32 imm_hi, imm_lo;
627 
628 	imm_hi = (imm7 & 0x38) >> 3;
629 	imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
630 	return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
631 }
632 
rvc_addi(u8 rd,u32 imm6)633 static inline u16 rvc_addi(u8 rd, u32 imm6)
634 {
635 	return rv_ci_insn(0, imm6, rd, 0x1);
636 }
637 
rvc_li(u8 rd,u32 imm6)638 static inline u16 rvc_li(u8 rd, u32 imm6)
639 {
640 	return rv_ci_insn(0x2, imm6, rd, 0x1);
641 }
642 
rvc_addi16sp(u32 imm10)643 static inline u16 rvc_addi16sp(u32 imm10)
644 {
645 	u32 imm;
646 
647 	imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
648 		((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
649 	return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
650 }
651 
rvc_lui(u8 rd,u32 imm6)652 static inline u16 rvc_lui(u8 rd, u32 imm6)
653 {
654 	return rv_ci_insn(0x3, imm6, rd, 0x1);
655 }
656 
rvc_srli(u8 rd,u32 imm6)657 static inline u16 rvc_srli(u8 rd, u32 imm6)
658 {
659 	return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
660 }
661 
rvc_srai(u8 rd,u32 imm6)662 static inline u16 rvc_srai(u8 rd, u32 imm6)
663 {
664 	return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
665 }
666 
rvc_andi(u8 rd,u32 imm6)667 static inline u16 rvc_andi(u8 rd, u32 imm6)
668 {
669 	return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
670 }
671 
rvc_sub(u8 rd,u8 rs)672 static inline u16 rvc_sub(u8 rd, u8 rs)
673 {
674 	return rv_ca_insn(0x23, rd, 0, rs, 0x1);
675 }
676 
rvc_xor(u8 rd,u8 rs)677 static inline u16 rvc_xor(u8 rd, u8 rs)
678 {
679 	return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
680 }
681 
rvc_or(u8 rd,u8 rs)682 static inline u16 rvc_or(u8 rd, u8 rs)
683 {
684 	return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
685 }
686 
rvc_and(u8 rd,u8 rs)687 static inline u16 rvc_and(u8 rd, u8 rs)
688 {
689 	return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
690 }
691 
rvc_slli(u8 rd,u32 imm6)692 static inline u16 rvc_slli(u8 rd, u32 imm6)
693 {
694 	return rv_ci_insn(0, imm6, rd, 0x2);
695 }
696 
rvc_lwsp(u8 rd,u32 imm8)697 static inline u16 rvc_lwsp(u8 rd, u32 imm8)
698 {
699 	u32 imm;
700 
701 	imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
702 	return rv_ci_insn(0x2, imm, rd, 0x2);
703 }
704 
rvc_jr(u8 rs1)705 static inline u16 rvc_jr(u8 rs1)
706 {
707 	return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
708 }
709 
rvc_mv(u8 rd,u8 rs)710 static inline u16 rvc_mv(u8 rd, u8 rs)
711 {
712 	return rv_cr_insn(0x8, rd, rs, 0x2);
713 }
714 
rvc_jalr(u8 rs1)715 static inline u16 rvc_jalr(u8 rs1)
716 {
717 	return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
718 }
719 
rvc_add(u8 rd,u8 rs)720 static inline u16 rvc_add(u8 rd, u8 rs)
721 {
722 	return rv_cr_insn(0x9, rd, rs, 0x2);
723 }
724 
rvc_swsp(u32 imm8,u8 rs2)725 static inline u16 rvc_swsp(u32 imm8, u8 rs2)
726 {
727 	u32 imm;
728 
729 	imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
730 	return rv_css_insn(0x6, imm, rs2, 0x2);
731 }
732 
733 /*
734  * RV64-only instructions.
735  *
736  * These instructions are not available on RV32.  Wrap them below a #if to
737  * ensure that the RV32 JIT doesn't emit any of these instructions.
738  */
739 
740 #if __riscv_xlen == 64
741 
rv_addiw(u8 rd,u8 rs1,u16 imm11_0)742 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
743 {
744 	return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
745 }
746 
rv_slliw(u8 rd,u8 rs1,u16 imm11_0)747 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
748 {
749 	return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
750 }
751 
rv_srliw(u8 rd,u8 rs1,u16 imm11_0)752 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
753 {
754 	return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
755 }
756 
rv_sraiw(u8 rd,u8 rs1,u16 imm11_0)757 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
758 {
759 	return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
760 }
761 
rv_addw(u8 rd,u8 rs1,u8 rs2)762 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
763 {
764 	return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
765 }
766 
rv_subw(u8 rd,u8 rs1,u8 rs2)767 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
768 {
769 	return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
770 }
771 
rv_sllw(u8 rd,u8 rs1,u8 rs2)772 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
773 {
774 	return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
775 }
776 
rv_srlw(u8 rd,u8 rs1,u8 rs2)777 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
778 {
779 	return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
780 }
781 
rv_sraw(u8 rd,u8 rs1,u8 rs2)782 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
783 {
784 	return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
785 }
786 
rv_mulw(u8 rd,u8 rs1,u8 rs2)787 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
788 {
789 	return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
790 }
791 
rv_divw(u8 rd,u8 rs1,u8 rs2)792 static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2)
793 {
794 	return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b);
795 }
796 
rv_divuw(u8 rd,u8 rs1,u8 rs2)797 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
798 {
799 	return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
800 }
801 
rv_remw(u8 rd,u8 rs1,u8 rs2)802 static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2)
803 {
804 	return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b);
805 }
806 
rv_remuw(u8 rd,u8 rs1,u8 rs2)807 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
808 {
809 	return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
810 }
811 
rv_ld(u8 rd,u16 imm11_0,u8 rs1)812 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
813 {
814 	return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
815 }
816 
rv_lwu(u8 rd,u16 imm11_0,u8 rs1)817 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
818 {
819 	return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
820 }
821 
rv_sd(u8 rs1,u16 imm11_0,u8 rs2)822 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
823 {
824 	return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
825 }
826 
rv_amoadd_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)827 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
828 {
829 	return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
830 }
831 
rv_amoand_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)832 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
833 {
834 	return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
835 }
836 
rv_amoor_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)837 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
838 {
839 	return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
840 }
841 
rv_amoxor_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)842 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
843 {
844 	return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
845 }
846 
rv_amoswap_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)847 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
848 {
849 	return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
850 }
851 
rv_lr_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)852 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
853 {
854 	return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
855 }
856 
rv_sc_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)857 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
858 {
859 	return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
860 }
861 
862 /* RV64-only RVC instructions. */
863 
rvc_ld(u8 rd,u32 imm8,u8 rs1)864 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
865 {
866 	u32 imm_hi, imm_lo;
867 
868 	imm_hi = (imm8 & 0x38) >> 3;
869 	imm_lo = (imm8 & 0xc0) >> 6;
870 	return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
871 }
872 
rvc_sd(u8 rs1,u32 imm8,u8 rs2)873 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
874 {
875 	u32 imm_hi, imm_lo;
876 
877 	imm_hi = (imm8 & 0x38) >> 3;
878 	imm_lo = (imm8 & 0xc0) >> 6;
879 	return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
880 }
881 
rvc_subw(u8 rd,u8 rs)882 static inline u16 rvc_subw(u8 rd, u8 rs)
883 {
884 	return rv_ca_insn(0x27, rd, 0, rs, 0x1);
885 }
886 
rvc_addiw(u8 rd,u32 imm6)887 static inline u16 rvc_addiw(u8 rd, u32 imm6)
888 {
889 	return rv_ci_insn(0x1, imm6, rd, 0x1);
890 }
891 
rvc_ldsp(u8 rd,u32 imm9)892 static inline u16 rvc_ldsp(u8 rd, u32 imm9)
893 {
894 	u32 imm;
895 
896 	imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
897 	return rv_ci_insn(0x3, imm, rd, 0x2);
898 }
899 
rvc_sdsp(u32 imm9,u8 rs2)900 static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
901 {
902 	u32 imm;
903 
904 	imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
905 	return rv_css_insn(0x7, imm, rs2, 0x2);
906 }
907 
908 #endif /* __riscv_xlen == 64 */
909 
910 /* Helper functions that emit RVC instructions when possible. */
911 
emit_jalr(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)912 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
913 {
914 	if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
915 		emitc(rvc_jalr(rs), ctx);
916 	else if (rvc_enabled() && !rd && rs && !imm)
917 		emitc(rvc_jr(rs), ctx);
918 	else
919 		emit(rv_jalr(rd, rs, imm), ctx);
920 }
921 
emit_mv(u8 rd,u8 rs,struct rv_jit_context * ctx)922 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
923 {
924 	if (rvc_enabled() && rd && rs)
925 		emitc(rvc_mv(rd, rs), ctx);
926 	else
927 		emit(rv_addi(rd, rs, 0), ctx);
928 }
929 
emit_add(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)930 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
931 {
932 	if (rvc_enabled() && rd && rd == rs1 && rs2)
933 		emitc(rvc_add(rd, rs2), ctx);
934 	else
935 		emit(rv_add(rd, rs1, rs2), ctx);
936 }
937 
emit_addi(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)938 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
939 {
940 	if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
941 		emitc(rvc_addi16sp(imm), ctx);
942 	else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
943 		 !(imm & 0x3) && imm)
944 		emitc(rvc_addi4spn(rd, imm), ctx);
945 	else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
946 		emitc(rvc_addi(rd, imm), ctx);
947 	else
948 		emit(rv_addi(rd, rs, imm), ctx);
949 }
950 
emit_li(u8 rd,s32 imm,struct rv_jit_context * ctx)951 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
952 {
953 	if (rvc_enabled() && rd && is_6b_int(imm))
954 		emitc(rvc_li(rd, imm), ctx);
955 	else
956 		emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
957 }
958 
emit_lui(u8 rd,s32 imm,struct rv_jit_context * ctx)959 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
960 {
961 	if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
962 		emitc(rvc_lui(rd, imm), ctx);
963 	else
964 		emit(rv_lui(rd, imm), ctx);
965 }
966 
emit_slli(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)967 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
968 {
969 	if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
970 		emitc(rvc_slli(rd, imm), ctx);
971 	else
972 		emit(rv_slli(rd, rs, imm), ctx);
973 }
974 
emit_andi(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)975 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
976 {
977 	if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
978 		emitc(rvc_andi(rd, imm), ctx);
979 	else
980 		emit(rv_andi(rd, rs, imm), ctx);
981 }
982 
emit_srli(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)983 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
984 {
985 	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
986 		emitc(rvc_srli(rd, imm), ctx);
987 	else
988 		emit(rv_srli(rd, rs, imm), ctx);
989 }
990 
emit_srai(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)991 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
992 {
993 	if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
994 		emitc(rvc_srai(rd, imm), ctx);
995 	else
996 		emit(rv_srai(rd, rs, imm), ctx);
997 }
998 
emit_sub(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)999 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1000 {
1001 	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1002 		emitc(rvc_sub(rd, rs2), ctx);
1003 	else
1004 		emit(rv_sub(rd, rs1, rs2), ctx);
1005 }
1006 
emit_or(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)1007 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1008 {
1009 	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1010 		emitc(rvc_or(rd, rs2), ctx);
1011 	else
1012 		emit(rv_or(rd, rs1, rs2), ctx);
1013 }
1014 
emit_and(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)1015 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1016 {
1017 	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1018 		emitc(rvc_and(rd, rs2), ctx);
1019 	else
1020 		emit(rv_and(rd, rs1, rs2), ctx);
1021 }
1022 
emit_xor(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)1023 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1024 {
1025 	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1026 		emitc(rvc_xor(rd, rs2), ctx);
1027 	else
1028 		emit(rv_xor(rd, rs1, rs2), ctx);
1029 }
1030 
emit_lw(u8 rd,s32 off,u8 rs1,struct rv_jit_context * ctx)1031 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1032 {
1033 	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
1034 		emitc(rvc_lwsp(rd, off), ctx);
1035 	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
1036 		emitc(rvc_lw(rd, off, rs1), ctx);
1037 	else
1038 		emit(rv_lw(rd, off, rs1), ctx);
1039 }
1040 
emit_sw(u8 rs1,s32 off,u8 rs2,struct rv_jit_context * ctx)1041 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1042 {
1043 	if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1044 		emitc(rvc_swsp(off, rs2), ctx);
1045 	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1046 		emitc(rvc_sw(rs1, off, rs2), ctx);
1047 	else
1048 		emit(rv_sw(rs1, off, rs2), ctx);
1049 }
1050 
1051 /* RV64-only helper functions. */
1052 #if __riscv_xlen == 64
1053 
emit_addiw(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)1054 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1055 {
1056 	if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1057 		emitc(rvc_addiw(rd, imm), ctx);
1058 	else
1059 		emit(rv_addiw(rd, rs, imm), ctx);
1060 }
1061 
emit_ld(u8 rd,s32 off,u8 rs1,struct rv_jit_context * ctx)1062 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1063 {
1064 	if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1065 		emitc(rvc_ldsp(rd, off), ctx);
1066 	else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1067 		emitc(rvc_ld(rd, off, rs1), ctx);
1068 	else
1069 		emit(rv_ld(rd, off, rs1), ctx);
1070 }
1071 
emit_sd(u8 rs1,s32 off,u8 rs2,struct rv_jit_context * ctx)1072 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1073 {
1074 	if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1075 		emitc(rvc_sdsp(off, rs2), ctx);
1076 	else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1077 		emitc(rvc_sd(rs1, off, rs2), ctx);
1078 	else
1079 		emit(rv_sd(rs1, off, rs2), ctx);
1080 }
1081 
emit_subw(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)1082 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1083 {
1084 	if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1085 		emitc(rvc_subw(rd, rs2), ctx);
1086 	else
1087 		emit(rv_subw(rd, rs1, rs2), ctx);
1088 }
1089 
1090 #endif /* __riscv_xlen == 64 */
1091 
1092 void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1093 void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1094 
1095 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1096 		      bool extra_pass);
1097 
1098 #endif /* _BPF_JIT_H */
1099