1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Common functionality for RV32 and RV64 BPF JIT compilers
4 *
5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com>
6 *
7 */
8
9 #ifndef _BPF_JIT_H
10 #define _BPF_JIT_H
11
12 #include <linux/bpf.h>
13 #include <linux/filter.h>
14 #include <asm/cacheflush.h>
15
rvc_enabled(void)16 static inline bool rvc_enabled(void)
17 {
18 return IS_ENABLED(CONFIG_RISCV_ISA_C);
19 }
20
21 enum {
22 RV_REG_ZERO = 0, /* The constant value 0 */
23 RV_REG_RA = 1, /* Return address */
24 RV_REG_SP = 2, /* Stack pointer */
25 RV_REG_GP = 3, /* Global pointer */
26 RV_REG_TP = 4, /* Thread pointer */
27 RV_REG_T0 = 5, /* Temporaries */
28 RV_REG_T1 = 6,
29 RV_REG_T2 = 7,
30 RV_REG_FP = 8, /* Saved register/frame pointer */
31 RV_REG_S1 = 9, /* Saved register */
32 RV_REG_A0 = 10, /* Function argument/return values */
33 RV_REG_A1 = 11, /* Function arguments */
34 RV_REG_A2 = 12,
35 RV_REG_A3 = 13,
36 RV_REG_A4 = 14,
37 RV_REG_A5 = 15,
38 RV_REG_A6 = 16,
39 RV_REG_A7 = 17,
40 RV_REG_S2 = 18, /* Saved registers */
41 RV_REG_S3 = 19,
42 RV_REG_S4 = 20,
43 RV_REG_S5 = 21,
44 RV_REG_S6 = 22,
45 RV_REG_S7 = 23,
46 RV_REG_S8 = 24,
47 RV_REG_S9 = 25,
48 RV_REG_S10 = 26,
49 RV_REG_S11 = 27,
50 RV_REG_T3 = 28, /* Temporaries */
51 RV_REG_T4 = 29,
52 RV_REG_T5 = 30,
53 RV_REG_T6 = 31,
54 };
55
is_creg(u8 reg)56 static inline bool is_creg(u8 reg)
57 {
58 return (1 << reg) & (BIT(RV_REG_FP) |
59 BIT(RV_REG_S1) |
60 BIT(RV_REG_A0) |
61 BIT(RV_REG_A1) |
62 BIT(RV_REG_A2) |
63 BIT(RV_REG_A3) |
64 BIT(RV_REG_A4) |
65 BIT(RV_REG_A5));
66 }
67
68 struct rv_jit_context {
69 struct bpf_prog *prog;
70 u16 *insns; /* RV insns */
71 int ninsns;
72 int epilogue_offset;
73 int *offset; /* BPF to RV */
74 int nexentries;
75 unsigned long flags;
76 int stack_size;
77 };
78
79 /* Convert from ninsns to bytes. */
ninsns_rvoff(int ninsns)80 static inline int ninsns_rvoff(int ninsns)
81 {
82 return ninsns << 1;
83 }
84
85 struct rv_jit_data {
86 struct bpf_binary_header *header;
87 u8 *image;
88 struct rv_jit_context ctx;
89 };
90
bpf_fill_ill_insns(void * area,unsigned int size)91 static inline void bpf_fill_ill_insns(void *area, unsigned int size)
92 {
93 memset(area, 0, size);
94 }
95
bpf_flush_icache(void * start,void * end)96 static inline void bpf_flush_icache(void *start, void *end)
97 {
98 flush_icache_range((unsigned long)start, (unsigned long)end);
99 }
100
101 /* Emit a 4-byte riscv instruction. */
emit(const u32 insn,struct rv_jit_context * ctx)102 static inline void emit(const u32 insn, struct rv_jit_context *ctx)
103 {
104 if (ctx->insns) {
105 ctx->insns[ctx->ninsns] = insn;
106 ctx->insns[ctx->ninsns + 1] = (insn >> 16);
107 }
108
109 ctx->ninsns += 2;
110 }
111
112 /* Emit a 2-byte riscv compressed instruction. */
emitc(const u16 insn,struct rv_jit_context * ctx)113 static inline void emitc(const u16 insn, struct rv_jit_context *ctx)
114 {
115 BUILD_BUG_ON(!rvc_enabled());
116
117 if (ctx->insns)
118 ctx->insns[ctx->ninsns] = insn;
119
120 ctx->ninsns++;
121 }
122
epilogue_offset(struct rv_jit_context * ctx)123 static inline int epilogue_offset(struct rv_jit_context *ctx)
124 {
125 int to = ctx->epilogue_offset, from = ctx->ninsns;
126
127 return ninsns_rvoff(to - from);
128 }
129
130 /* Return -1 or inverted cond. */
invert_bpf_cond(u8 cond)131 static inline int invert_bpf_cond(u8 cond)
132 {
133 switch (cond) {
134 case BPF_JEQ:
135 return BPF_JNE;
136 case BPF_JGT:
137 return BPF_JLE;
138 case BPF_JLT:
139 return BPF_JGE;
140 case BPF_JGE:
141 return BPF_JLT;
142 case BPF_JLE:
143 return BPF_JGT;
144 case BPF_JNE:
145 return BPF_JEQ;
146 case BPF_JSGT:
147 return BPF_JSLE;
148 case BPF_JSLT:
149 return BPF_JSGE;
150 case BPF_JSGE:
151 return BPF_JSLT;
152 case BPF_JSLE:
153 return BPF_JSGT;
154 }
155 return -1;
156 }
157
is_6b_int(long val)158 static inline bool is_6b_int(long val)
159 {
160 return -(1L << 5) <= val && val < (1L << 5);
161 }
162
is_7b_uint(unsigned long val)163 static inline bool is_7b_uint(unsigned long val)
164 {
165 return val < (1UL << 7);
166 }
167
is_8b_uint(unsigned long val)168 static inline bool is_8b_uint(unsigned long val)
169 {
170 return val < (1UL << 8);
171 }
172
is_9b_uint(unsigned long val)173 static inline bool is_9b_uint(unsigned long val)
174 {
175 return val < (1UL << 9);
176 }
177
is_10b_int(long val)178 static inline bool is_10b_int(long val)
179 {
180 return -(1L << 9) <= val && val < (1L << 9);
181 }
182
is_10b_uint(unsigned long val)183 static inline bool is_10b_uint(unsigned long val)
184 {
185 return val < (1UL << 10);
186 }
187
is_12b_int(long val)188 static inline bool is_12b_int(long val)
189 {
190 return -(1L << 11) <= val && val < (1L << 11);
191 }
192
is_12b_check(int off,int insn)193 static inline int is_12b_check(int off, int insn)
194 {
195 if (!is_12b_int(off)) {
196 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n",
197 insn, (int)off);
198 return -1;
199 }
200 return 0;
201 }
202
is_13b_int(long val)203 static inline bool is_13b_int(long val)
204 {
205 return -(1L << 12) <= val && val < (1L << 12);
206 }
207
is_21b_int(long val)208 static inline bool is_21b_int(long val)
209 {
210 return -(1L << 20) <= val && val < (1L << 20);
211 }
212
rv_offset(int insn,int off,struct rv_jit_context * ctx)213 static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx)
214 {
215 int from, to;
216
217 off++; /* BPF branch is from PC+1, RV is from PC */
218 from = (insn > 0) ? ctx->offset[insn - 1] : 0;
219 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : 0;
220 return ninsns_rvoff(to - from);
221 }
222
223 /* Instruction formats. */
224
rv_r_insn(u8 funct7,u8 rs2,u8 rs1,u8 funct3,u8 rd,u8 opcode)225 static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd,
226 u8 opcode)
227 {
228 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
229 (rd << 7) | opcode;
230 }
231
rv_i_insn(u16 imm11_0,u8 rs1,u8 funct3,u8 rd,u8 opcode)232 static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode)
233 {
234 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) |
235 opcode;
236 }
237
rv_s_insn(u16 imm11_0,u8 rs2,u8 rs1,u8 funct3,u8 opcode)238 static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
239 {
240 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f;
241
242 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
243 (imm4_0 << 7) | opcode;
244 }
245
rv_b_insn(u16 imm12_1,u8 rs2,u8 rs1,u8 funct3,u8 opcode)246 static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode)
247 {
248 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4);
249 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10);
250
251 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) |
252 (imm4_1 << 7) | opcode;
253 }
254
rv_u_insn(u32 imm31_12,u8 rd,u8 opcode)255 static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode)
256 {
257 return (imm31_12 << 12) | (rd << 7) | opcode;
258 }
259
rv_j_insn(u32 imm20_1,u8 rd,u8 opcode)260 static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode)
261 {
262 u32 imm;
263
264 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) |
265 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11);
266
267 return (imm << 12) | (rd << 7) | opcode;
268 }
269
rv_amo_insn(u8 funct5,u8 aq,u8 rl,u8 rs2,u8 rs1,u8 funct3,u8 rd,u8 opcode)270 static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1,
271 u8 funct3, u8 rd, u8 opcode)
272 {
273 u8 funct7 = (funct5 << 2) | (aq << 1) | rl;
274
275 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode);
276 }
277
278 /* RISC-V compressed instruction formats. */
279
rv_cr_insn(u8 funct4,u8 rd,u8 rs2,u8 op)280 static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op)
281 {
282 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op;
283 }
284
rv_ci_insn(u8 funct3,u32 imm6,u8 rd,u8 op)285 static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op)
286 {
287 u32 imm;
288
289 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
290 return (funct3 << 13) | (rd << 7) | op | imm;
291 }
292
rv_css_insn(u8 funct3,u32 uimm,u8 rs2,u8 op)293 static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op)
294 {
295 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op;
296 }
297
rv_ciw_insn(u8 funct3,u32 uimm,u8 rd,u8 op)298 static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op)
299 {
300 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op;
301 }
302
rv_cl_insn(u8 funct3,u32 imm_hi,u8 rs1,u32 imm_lo,u8 rd,u8 op)303 static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd,
304 u8 op)
305 {
306 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
307 (imm_lo << 5) | ((rd & 0x7) << 2) | op;
308 }
309
rv_cs_insn(u8 funct3,u32 imm_hi,u8 rs1,u32 imm_lo,u8 rs2,u8 op)310 static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2,
311 u8 op)
312 {
313 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) |
314 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op;
315 }
316
rv_ca_insn(u8 funct6,u8 rd,u8 funct2,u8 rs2,u8 op)317 static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op)
318 {
319 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) |
320 ((rs2 & 0x7) << 2) | op;
321 }
322
rv_cb_insn(u8 funct3,u32 imm6,u8 funct2,u8 rd,u8 op)323 static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op)
324 {
325 u32 imm;
326
327 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2);
328 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm;
329 }
330
331 /* Instructions shared by both RV32 and RV64. */
332
rv_addi(u8 rd,u8 rs1,u16 imm11_0)333 static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0)
334 {
335 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13);
336 }
337
rv_andi(u8 rd,u8 rs1,u16 imm11_0)338 static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0)
339 {
340 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13);
341 }
342
rv_ori(u8 rd,u8 rs1,u16 imm11_0)343 static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0)
344 {
345 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13);
346 }
347
rv_xori(u8 rd,u8 rs1,u16 imm11_0)348 static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0)
349 {
350 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13);
351 }
352
rv_slli(u8 rd,u8 rs1,u16 imm11_0)353 static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0)
354 {
355 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13);
356 }
357
rv_srli(u8 rd,u8 rs1,u16 imm11_0)358 static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0)
359 {
360 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13);
361 }
362
rv_srai(u8 rd,u8 rs1,u16 imm11_0)363 static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0)
364 {
365 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13);
366 }
367
rv_lui(u8 rd,u32 imm31_12)368 static inline u32 rv_lui(u8 rd, u32 imm31_12)
369 {
370 return rv_u_insn(imm31_12, rd, 0x37);
371 }
372
rv_auipc(u8 rd,u32 imm31_12)373 static inline u32 rv_auipc(u8 rd, u32 imm31_12)
374 {
375 return rv_u_insn(imm31_12, rd, 0x17);
376 }
377
rv_add(u8 rd,u8 rs1,u8 rs2)378 static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2)
379 {
380 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33);
381 }
382
rv_sub(u8 rd,u8 rs1,u8 rs2)383 static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2)
384 {
385 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33);
386 }
387
rv_sltu(u8 rd,u8 rs1,u8 rs2)388 static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2)
389 {
390 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33);
391 }
392
rv_and(u8 rd,u8 rs1,u8 rs2)393 static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2)
394 {
395 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33);
396 }
397
rv_or(u8 rd,u8 rs1,u8 rs2)398 static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2)
399 {
400 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33);
401 }
402
rv_xor(u8 rd,u8 rs1,u8 rs2)403 static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2)
404 {
405 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33);
406 }
407
rv_sll(u8 rd,u8 rs1,u8 rs2)408 static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2)
409 {
410 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33);
411 }
412
rv_srl(u8 rd,u8 rs1,u8 rs2)413 static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2)
414 {
415 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33);
416 }
417
rv_sra(u8 rd,u8 rs1,u8 rs2)418 static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2)
419 {
420 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33);
421 }
422
rv_mul(u8 rd,u8 rs1,u8 rs2)423 static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2)
424 {
425 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33);
426 }
427
rv_mulhu(u8 rd,u8 rs1,u8 rs2)428 static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2)
429 {
430 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33);
431 }
432
rv_divu(u8 rd,u8 rs1,u8 rs2)433 static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2)
434 {
435 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33);
436 }
437
rv_remu(u8 rd,u8 rs1,u8 rs2)438 static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2)
439 {
440 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33);
441 }
442
rv_jal(u8 rd,u32 imm20_1)443 static inline u32 rv_jal(u8 rd, u32 imm20_1)
444 {
445 return rv_j_insn(imm20_1, rd, 0x6f);
446 }
447
rv_jalr(u8 rd,u8 rs1,u16 imm11_0)448 static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0)
449 {
450 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67);
451 }
452
rv_beq(u8 rs1,u8 rs2,u16 imm12_1)453 static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1)
454 {
455 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63);
456 }
457
rv_bne(u8 rs1,u8 rs2,u16 imm12_1)458 static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1)
459 {
460 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63);
461 }
462
rv_bltu(u8 rs1,u8 rs2,u16 imm12_1)463 static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1)
464 {
465 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63);
466 }
467
rv_bgtu(u8 rs1,u8 rs2,u16 imm12_1)468 static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1)
469 {
470 return rv_bltu(rs2, rs1, imm12_1);
471 }
472
rv_bgeu(u8 rs1,u8 rs2,u16 imm12_1)473 static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1)
474 {
475 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63);
476 }
477
rv_bleu(u8 rs1,u8 rs2,u16 imm12_1)478 static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1)
479 {
480 return rv_bgeu(rs2, rs1, imm12_1);
481 }
482
rv_blt(u8 rs1,u8 rs2,u16 imm12_1)483 static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1)
484 {
485 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63);
486 }
487
rv_bgt(u8 rs1,u8 rs2,u16 imm12_1)488 static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1)
489 {
490 return rv_blt(rs2, rs1, imm12_1);
491 }
492
rv_bge(u8 rs1,u8 rs2,u16 imm12_1)493 static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1)
494 {
495 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63);
496 }
497
rv_ble(u8 rs1,u8 rs2,u16 imm12_1)498 static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1)
499 {
500 return rv_bge(rs2, rs1, imm12_1);
501 }
502
rv_lw(u8 rd,u16 imm11_0,u8 rs1)503 static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1)
504 {
505 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03);
506 }
507
rv_lbu(u8 rd,u16 imm11_0,u8 rs1)508 static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1)
509 {
510 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03);
511 }
512
rv_lhu(u8 rd,u16 imm11_0,u8 rs1)513 static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1)
514 {
515 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03);
516 }
517
rv_sb(u8 rs1,u16 imm11_0,u8 rs2)518 static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2)
519 {
520 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23);
521 }
522
rv_sh(u8 rs1,u16 imm11_0,u8 rs2)523 static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2)
524 {
525 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23);
526 }
527
rv_sw(u8 rs1,u16 imm11_0,u8 rs2)528 static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2)
529 {
530 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23);
531 }
532
rv_amoadd_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)533 static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
534 {
535 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f);
536 }
537
rv_amoand_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)538 static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
539 {
540 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f);
541 }
542
rv_amoor_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)543 static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
544 {
545 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f);
546 }
547
rv_amoxor_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)548 static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
549 {
550 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f);
551 }
552
rv_amoswap_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)553 static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
554 {
555 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f);
556 }
557
rv_lr_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)558 static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
559 {
560 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f);
561 }
562
rv_sc_w(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)563 static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
564 {
565 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f);
566 }
567
rv_fence(u8 pred,u8 succ)568 static inline u32 rv_fence(u8 pred, u8 succ)
569 {
570 u16 imm11_0 = pred << 4 | succ;
571
572 return rv_i_insn(imm11_0, 0, 0, 0, 0xf);
573 }
574
575 /* RVC instrutions. */
576
rvc_addi4spn(u8 rd,u32 imm10)577 static inline u16 rvc_addi4spn(u8 rd, u32 imm10)
578 {
579 u32 imm;
580
581 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) |
582 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3);
583 return rv_ciw_insn(0x0, imm, rd, 0x0);
584 }
585
rvc_lw(u8 rd,u32 imm7,u8 rs1)586 static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1)
587 {
588 u32 imm_hi, imm_lo;
589
590 imm_hi = (imm7 & 0x38) >> 3;
591 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
592 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0);
593 }
594
rvc_sw(u8 rs1,u32 imm7,u8 rs2)595 static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2)
596 {
597 u32 imm_hi, imm_lo;
598
599 imm_hi = (imm7 & 0x38) >> 3;
600 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6);
601 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0);
602 }
603
rvc_addi(u8 rd,u32 imm6)604 static inline u16 rvc_addi(u8 rd, u32 imm6)
605 {
606 return rv_ci_insn(0, imm6, rd, 0x1);
607 }
608
rvc_li(u8 rd,u32 imm6)609 static inline u16 rvc_li(u8 rd, u32 imm6)
610 {
611 return rv_ci_insn(0x2, imm6, rd, 0x1);
612 }
613
rvc_addi16sp(u32 imm10)614 static inline u16 rvc_addi16sp(u32 imm10)
615 {
616 u32 imm;
617
618 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) |
619 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5);
620 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1);
621 }
622
rvc_lui(u8 rd,u32 imm6)623 static inline u16 rvc_lui(u8 rd, u32 imm6)
624 {
625 return rv_ci_insn(0x3, imm6, rd, 0x1);
626 }
627
rvc_srli(u8 rd,u32 imm6)628 static inline u16 rvc_srli(u8 rd, u32 imm6)
629 {
630 return rv_cb_insn(0x4, imm6, 0, rd, 0x1);
631 }
632
rvc_srai(u8 rd,u32 imm6)633 static inline u16 rvc_srai(u8 rd, u32 imm6)
634 {
635 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1);
636 }
637
rvc_andi(u8 rd,u32 imm6)638 static inline u16 rvc_andi(u8 rd, u32 imm6)
639 {
640 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1);
641 }
642
rvc_sub(u8 rd,u8 rs)643 static inline u16 rvc_sub(u8 rd, u8 rs)
644 {
645 return rv_ca_insn(0x23, rd, 0, rs, 0x1);
646 }
647
rvc_xor(u8 rd,u8 rs)648 static inline u16 rvc_xor(u8 rd, u8 rs)
649 {
650 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1);
651 }
652
rvc_or(u8 rd,u8 rs)653 static inline u16 rvc_or(u8 rd, u8 rs)
654 {
655 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1);
656 }
657
rvc_and(u8 rd,u8 rs)658 static inline u16 rvc_and(u8 rd, u8 rs)
659 {
660 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1);
661 }
662
rvc_slli(u8 rd,u32 imm6)663 static inline u16 rvc_slli(u8 rd, u32 imm6)
664 {
665 return rv_ci_insn(0, imm6, rd, 0x2);
666 }
667
rvc_lwsp(u8 rd,u32 imm8)668 static inline u16 rvc_lwsp(u8 rd, u32 imm8)
669 {
670 u32 imm;
671
672 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c);
673 return rv_ci_insn(0x2, imm, rd, 0x2);
674 }
675
rvc_jr(u8 rs1)676 static inline u16 rvc_jr(u8 rs1)
677 {
678 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2);
679 }
680
rvc_mv(u8 rd,u8 rs)681 static inline u16 rvc_mv(u8 rd, u8 rs)
682 {
683 return rv_cr_insn(0x8, rd, rs, 0x2);
684 }
685
rvc_jalr(u8 rs1)686 static inline u16 rvc_jalr(u8 rs1)
687 {
688 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2);
689 }
690
rvc_add(u8 rd,u8 rs)691 static inline u16 rvc_add(u8 rd, u8 rs)
692 {
693 return rv_cr_insn(0x9, rd, rs, 0x2);
694 }
695
rvc_swsp(u32 imm8,u8 rs2)696 static inline u16 rvc_swsp(u32 imm8, u8 rs2)
697 {
698 u32 imm;
699
700 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6);
701 return rv_css_insn(0x6, imm, rs2, 0x2);
702 }
703
704 /*
705 * RV64-only instructions.
706 *
707 * These instructions are not available on RV32. Wrap them below a #if to
708 * ensure that the RV32 JIT doesn't emit any of these instructions.
709 */
710
711 #if __riscv_xlen == 64
712
rv_addiw(u8 rd,u8 rs1,u16 imm11_0)713 static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0)
714 {
715 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b);
716 }
717
rv_slliw(u8 rd,u8 rs1,u16 imm11_0)718 static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0)
719 {
720 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b);
721 }
722
rv_srliw(u8 rd,u8 rs1,u16 imm11_0)723 static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0)
724 {
725 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b);
726 }
727
rv_sraiw(u8 rd,u8 rs1,u16 imm11_0)728 static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0)
729 {
730 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b);
731 }
732
rv_addw(u8 rd,u8 rs1,u8 rs2)733 static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2)
734 {
735 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b);
736 }
737
rv_subw(u8 rd,u8 rs1,u8 rs2)738 static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2)
739 {
740 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b);
741 }
742
rv_sllw(u8 rd,u8 rs1,u8 rs2)743 static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2)
744 {
745 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b);
746 }
747
rv_srlw(u8 rd,u8 rs1,u8 rs2)748 static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2)
749 {
750 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b);
751 }
752
rv_sraw(u8 rd,u8 rs1,u8 rs2)753 static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2)
754 {
755 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b);
756 }
757
rv_mulw(u8 rd,u8 rs1,u8 rs2)758 static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2)
759 {
760 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b);
761 }
762
rv_divuw(u8 rd,u8 rs1,u8 rs2)763 static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2)
764 {
765 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b);
766 }
767
rv_remuw(u8 rd,u8 rs1,u8 rs2)768 static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2)
769 {
770 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b);
771 }
772
rv_ld(u8 rd,u16 imm11_0,u8 rs1)773 static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1)
774 {
775 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03);
776 }
777
rv_lwu(u8 rd,u16 imm11_0,u8 rs1)778 static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1)
779 {
780 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03);
781 }
782
rv_sd(u8 rs1,u16 imm11_0,u8 rs2)783 static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2)
784 {
785 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23);
786 }
787
rv_amoadd_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)788 static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
789 {
790 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f);
791 }
792
rv_amoand_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)793 static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
794 {
795 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f);
796 }
797
rv_amoor_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)798 static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
799 {
800 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f);
801 }
802
rv_amoxor_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)803 static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
804 {
805 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f);
806 }
807
rv_amoswap_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)808 static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
809 {
810 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f);
811 }
812
rv_lr_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)813 static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
814 {
815 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f);
816 }
817
rv_sc_d(u8 rd,u8 rs2,u8 rs1,u8 aq,u8 rl)818 static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl)
819 {
820 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f);
821 }
822
823 /* RV64-only RVC instructions. */
824
rvc_ld(u8 rd,u32 imm8,u8 rs1)825 static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1)
826 {
827 u32 imm_hi, imm_lo;
828
829 imm_hi = (imm8 & 0x38) >> 3;
830 imm_lo = (imm8 & 0xc0) >> 6;
831 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0);
832 }
833
rvc_sd(u8 rs1,u32 imm8,u8 rs2)834 static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2)
835 {
836 u32 imm_hi, imm_lo;
837
838 imm_hi = (imm8 & 0x38) >> 3;
839 imm_lo = (imm8 & 0xc0) >> 6;
840 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0);
841 }
842
rvc_subw(u8 rd,u8 rs)843 static inline u16 rvc_subw(u8 rd, u8 rs)
844 {
845 return rv_ca_insn(0x27, rd, 0, rs, 0x1);
846 }
847
rvc_addiw(u8 rd,u32 imm6)848 static inline u16 rvc_addiw(u8 rd, u32 imm6)
849 {
850 return rv_ci_insn(0x1, imm6, rd, 0x1);
851 }
852
rvc_ldsp(u8 rd,u32 imm9)853 static inline u16 rvc_ldsp(u8 rd, u32 imm9)
854 {
855 u32 imm;
856
857 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38);
858 return rv_ci_insn(0x3, imm, rd, 0x2);
859 }
860
rvc_sdsp(u32 imm9,u8 rs2)861 static inline u16 rvc_sdsp(u32 imm9, u8 rs2)
862 {
863 u32 imm;
864
865 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6);
866 return rv_css_insn(0x7, imm, rs2, 0x2);
867 }
868
869 #endif /* __riscv_xlen == 64 */
870
871 /* Helper functions that emit RVC instructions when possible. */
872
emit_jalr(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)873 static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
874 {
875 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm)
876 emitc(rvc_jalr(rs), ctx);
877 else if (rvc_enabled() && !rd && rs && !imm)
878 emitc(rvc_jr(rs), ctx);
879 else
880 emit(rv_jalr(rd, rs, imm), ctx);
881 }
882
emit_mv(u8 rd,u8 rs,struct rv_jit_context * ctx)883 static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx)
884 {
885 if (rvc_enabled() && rd && rs)
886 emitc(rvc_mv(rd, rs), ctx);
887 else
888 emit(rv_addi(rd, rs, 0), ctx);
889 }
890
emit_add(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)891 static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
892 {
893 if (rvc_enabled() && rd && rd == rs1 && rs2)
894 emitc(rvc_add(rd, rs2), ctx);
895 else
896 emit(rv_add(rd, rs1, rs2), ctx);
897 }
898
emit_addi(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)899 static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
900 {
901 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf))
902 emitc(rvc_addi16sp(imm), ctx);
903 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) &&
904 !(imm & 0x3) && imm)
905 emitc(rvc_addi4spn(rd, imm), ctx);
906 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm))
907 emitc(rvc_addi(rd, imm), ctx);
908 else
909 emit(rv_addi(rd, rs, imm), ctx);
910 }
911
emit_li(u8 rd,s32 imm,struct rv_jit_context * ctx)912 static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx)
913 {
914 if (rvc_enabled() && rd && is_6b_int(imm))
915 emitc(rvc_li(rd, imm), ctx);
916 else
917 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx);
918 }
919
emit_lui(u8 rd,s32 imm,struct rv_jit_context * ctx)920 static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx)
921 {
922 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm)
923 emitc(rvc_lui(rd, imm), ctx);
924 else
925 emit(rv_lui(rd, imm), ctx);
926 }
927
emit_slli(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)928 static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
929 {
930 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen)
931 emitc(rvc_slli(rd, imm), ctx);
932 else
933 emit(rv_slli(rd, rs, imm), ctx);
934 }
935
emit_andi(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)936 static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
937 {
938 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm))
939 emitc(rvc_andi(rd, imm), ctx);
940 else
941 emit(rv_andi(rd, rs, imm), ctx);
942 }
943
emit_srli(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)944 static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
945 {
946 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
947 emitc(rvc_srli(rd, imm), ctx);
948 else
949 emit(rv_srli(rd, rs, imm), ctx);
950 }
951
emit_srai(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)952 static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
953 {
954 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen)
955 emitc(rvc_srai(rd, imm), ctx);
956 else
957 emit(rv_srai(rd, rs, imm), ctx);
958 }
959
emit_sub(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)960 static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
961 {
962 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
963 emitc(rvc_sub(rd, rs2), ctx);
964 else
965 emit(rv_sub(rd, rs1, rs2), ctx);
966 }
967
emit_or(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)968 static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
969 {
970 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
971 emitc(rvc_or(rd, rs2), ctx);
972 else
973 emit(rv_or(rd, rs1, rs2), ctx);
974 }
975
emit_and(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)976 static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
977 {
978 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
979 emitc(rvc_and(rd, rs2), ctx);
980 else
981 emit(rv_and(rd, rs1, rs2), ctx);
982 }
983
emit_xor(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)984 static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
985 {
986 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
987 emitc(rvc_xor(rd, rs2), ctx);
988 else
989 emit(rv_xor(rd, rs1, rs2), ctx);
990 }
991
emit_lw(u8 rd,s32 off,u8 rs1,struct rv_jit_context * ctx)992 static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
993 {
994 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3))
995 emitc(rvc_lwsp(rd, off), ctx);
996 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3))
997 emitc(rvc_lw(rd, off, rs1), ctx);
998 else
999 emit(rv_lw(rd, off, rs1), ctx);
1000 }
1001
emit_sw(u8 rs1,s32 off,u8 rs2,struct rv_jit_context * ctx)1002 static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1003 {
1004 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3))
1005 emitc(rvc_swsp(off, rs2), ctx);
1006 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3))
1007 emitc(rvc_sw(rs1, off, rs2), ctx);
1008 else
1009 emit(rv_sw(rs1, off, rs2), ctx);
1010 }
1011
1012 /* RV64-only helper functions. */
1013 #if __riscv_xlen == 64
1014
emit_addiw(u8 rd,u8 rs,s32 imm,struct rv_jit_context * ctx)1015 static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx)
1016 {
1017 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm))
1018 emitc(rvc_addiw(rd, imm), ctx);
1019 else
1020 emit(rv_addiw(rd, rs, imm), ctx);
1021 }
1022
emit_ld(u8 rd,s32 off,u8 rs1,struct rv_jit_context * ctx)1023 static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx)
1024 {
1025 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7))
1026 emitc(rvc_ldsp(rd, off), ctx);
1027 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7))
1028 emitc(rvc_ld(rd, off, rs1), ctx);
1029 else
1030 emit(rv_ld(rd, off, rs1), ctx);
1031 }
1032
emit_sd(u8 rs1,s32 off,u8 rs2,struct rv_jit_context * ctx)1033 static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
1034 {
1035 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7))
1036 emitc(rvc_sdsp(off, rs2), ctx);
1037 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7))
1038 emitc(rvc_sd(rs1, off, rs2), ctx);
1039 else
1040 emit(rv_sd(rs1, off, rs2), ctx);
1041 }
1042
emit_subw(u8 rd,u8 rs1,u8 rs2,struct rv_jit_context * ctx)1043 static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
1044 {
1045 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2))
1046 emitc(rvc_subw(rd, rs2), ctx);
1047 else
1048 emit(rv_subw(rd, rs1, rs2), ctx);
1049 }
1050
1051 #endif /* __riscv_xlen == 64 */
1052
1053 void bpf_jit_build_prologue(struct rv_jit_context *ctx);
1054 void bpf_jit_build_epilogue(struct rv_jit_context *ctx);
1055
1056 int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
1057 bool extra_pass);
1058
1059 #endif /* _BPF_JIT_H */
1060