Lines Matching refs:mem
11 mem,
80 ( $mem:ident, $data:tt, $t:ty ) => {{
81 let size = mem::size_of::<$t>() as usize;
82 assert!($mem.offset + size <= $mem.contents.len());
84 let mut ptr = $mem.contents.as_ptr().add($mem.offset) as *mut $t;
87 $mem.offset += size;
113 fn emit1(&self, mem: &mut JitMemory, data: u8) { in emit1()
114 emit_bytes!(mem, data, u8); in emit1()
117 fn emit2(&self, mem: &mut JitMemory, data: u16) { in emit2()
118 emit_bytes!(mem, data, u16); in emit2()
121 fn emit4(&self, mem: &mut JitMemory, data: u32) { in emit4()
122 emit_bytes!(mem, data, u32); in emit4()
125 fn emit8(&self, mem: &mut JitMemory, data: u64) { in emit8()
126 emit_bytes!(mem, data, u64); in emit8()
129 fn emit_modrm(&self, mem: &mut JitMemory, modrm: u8, r: u8, m: u8) { in emit_modrm()
131 self.emit1(mem, (modrm & 0xc0) | ((r & 0b111) << 3) | (m & 0b111)); in emit_modrm()
134 fn emit_modrm_reg2reg(&self, mem: &mut JitMemory, r: u8, m: u8) { in emit_modrm_reg2reg()
135 self.emit_modrm(mem, 0xc0, r, m); in emit_modrm_reg2reg()
138 fn emit_modrm_and_displacement(&self, mem: &mut JitMemory, r: u8, m: u8, d: i32) { in emit_modrm_and_displacement()
140 self.emit_modrm(mem, 0x00, r, m); in emit_modrm_and_displacement()
142 self.emit_modrm(mem, 0x40, r, m); in emit_modrm_and_displacement()
143 self.emit1(mem, d as u8); in emit_modrm_and_displacement()
145 self.emit_modrm(mem, 0x80, r, m); in emit_modrm_and_displacement()
146 self.emit4(mem, d as u32); in emit_modrm_and_displacement()
154 fn emit_rex(&self, mem: &mut JitMemory, w: u8, r: u8, x: u8, b: u8) { in emit_rex()
159 self.emit1(mem, 0x40 | (w << 3) | (r << 2) | (x << 1) | b); in emit_rex()
164 fn emit_basic_rex(&self, mem: &mut JitMemory, w: u8, src: u8, dst: u8) { in emit_basic_rex()
170 self.emit_rex(mem, w, is_masked(src, 8), 0, is_masked(dst, 8)); in emit_basic_rex()
174 fn emit_push(&self, mem: &mut JitMemory, r: u8) { in emit_push()
175 self.emit_basic_rex(mem, 0, 0, r); in emit_push()
176 self.emit1(mem, 0x50 | (r & 0b111)); in emit_push()
179 fn emit_pop(&self, mem: &mut JitMemory, r: u8) { in emit_pop()
180 self.emit_basic_rex(mem, 0, 0, r); in emit_pop()
181 self.emit1(mem, 0x58 | (r & 0b111)); in emit_pop()
187 fn emit_alu32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { in emit_alu32()
188 self.emit_basic_rex(mem, 0, src, dst); in emit_alu32()
189 self.emit1(mem, op); in emit_alu32()
190 self.emit_modrm_reg2reg(mem, src, dst); in emit_alu32()
194 fn emit_alu32_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { in emit_alu32_imm32()
195 self.emit_alu32(mem, op, src, dst); in emit_alu32_imm32()
196 self.emit4(mem, imm as u32); in emit_alu32_imm32()
200 fn emit_alu32_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { in emit_alu32_imm8()
201 self.emit_alu32(mem, op, src, dst); in emit_alu32_imm8()
202 self.emit1(mem, imm as u8); in emit_alu32_imm8()
208 fn emit_alu64(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8) { in emit_alu64()
209 self.emit_basic_rex(mem, 1, src, dst); in emit_alu64()
210 self.emit1(mem, op); in emit_alu64()
211 self.emit_modrm_reg2reg(mem, src, dst); in emit_alu64()
215 fn emit_alu64_imm32(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i32) { in emit_alu64_imm32()
216 self.emit_alu64(mem, op, src, dst); in emit_alu64_imm32()
217 self.emit4(mem, imm as u32); in emit_alu64_imm32()
221 fn emit_alu64_imm8(&self, mem: &mut JitMemory, op: u8, src: u8, dst: u8, imm: i8) { in emit_alu64_imm8()
222 self.emit_alu64(mem, op, src, dst); in emit_alu64_imm8()
223 self.emit1(mem, imm as u8); in emit_alu64_imm8()
227 fn emit_mov(&self, mem: &mut JitMemory, src: u8, dst: u8) { in emit_mov()
228 self.emit_alu64(mem, 0x89, src, dst); in emit_mov()
231 fn emit_cmp_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { in emit_cmp_imm32()
232 self.emit_alu64_imm32(mem, 0x81, 7, dst, imm); in emit_cmp_imm32()
235 fn emit_cmp(&self, mem: &mut JitMemory, src: u8, dst: u8) { in emit_cmp()
236 self.emit_alu64(mem, 0x39, src, dst); in emit_cmp()
239 fn emit_cmp32_imm32(&self, mem: &mut JitMemory, dst: u8, imm: i32) { in emit_cmp32_imm32()
240 self.emit_alu32_imm32(mem, 0x81, 7, dst, imm); in emit_cmp32_imm32()
243 fn emit_cmp32(&self, mem: &mut JitMemory, src: u8, dst: u8) { in emit_cmp32()
244 self.emit_alu32(mem, 0x39, src, dst); in emit_cmp32()
248 fn emit_load(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { in emit_load()
253 self.emit_basic_rex(mem, data, dst, src); in emit_load()
258 self.emit1(mem, 0x0f); in emit_load()
259 self.emit1(mem, 0xb6); in emit_load()
263 self.emit1(mem, 0x0f); in emit_load()
264 self.emit1(mem, 0xb7); in emit_load()
268 self.emit1(mem, 0x8b); in emit_load()
272 self.emit_modrm_and_displacement(mem, dst, src, offset); in emit_load()
276 fn emit_load_imm(&self, mem: &mut JitMemory, dst: u8, imm: i64) { in emit_load_imm()
278 self.emit_alu64_imm32(mem, 0xc7, 0, dst, imm as i32); in emit_load_imm()
281 self.emit_basic_rex(mem, 1, 0, dst); in emit_load_imm()
282 self.emit1(mem, 0xb8 | (dst & 0b111)); in emit_load_imm()
283 self.emit8(mem, imm as u64); in emit_load_imm()
288 fn emit_store(&self, mem: &mut JitMemory, size: OperandSize, src: u8, dst: u8, offset: i32) { in emit_store()
290 OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override in emit_store()
303 self.emit_rex(mem, rexw, is_masked(src, 8), 0, is_masked(dst, 8)); in emit_store()
306 OperandSize::S8 => self.emit1(mem, 0x88), in emit_store()
307 _ => self.emit1(mem, 0x89), in emit_store()
309 self.emit_modrm_and_displacement(mem, src, dst, offset); in emit_store()
315 mem: &mut JitMemory, in emit_store_imm32()
322 OperandSize::S16 => self.emit1(mem, 0x66), // 16-bit override in emit_store_imm32()
326 OperandSize::S64 => self.emit_basic_rex(mem, 1, 0, dst), in emit_store_imm32()
327 _ => self.emit_basic_rex(mem, 0, 0, dst), in emit_store_imm32()
330 OperandSize::S8 => self.emit1(mem, 0xc6), in emit_store_imm32()
331 _ => self.emit1(mem, 0xc7), in emit_store_imm32()
333 self.emit_modrm_and_displacement(mem, 0, dst, offset); in emit_store_imm32()
335 OperandSize::S8 => self.emit1(mem, imm as u8), in emit_store_imm32()
336 OperandSize::S16 => self.emit2(mem, imm as u16), in emit_store_imm32()
337 _ => self.emit4(mem, imm as u32), in emit_store_imm32()
341 fn emit_direct_jcc(&self, mem: &mut JitMemory, code: u8, offset: u32) { in emit_direct_jcc()
342 self.emit1(mem, 0x0f); in emit_direct_jcc()
343 self.emit1(mem, code); in emit_direct_jcc()
344 emit_bytes!(mem, offset, u32); in emit_direct_jcc()
347 fn emit_call(&self, mem: &mut JitMemory, target: usize) { in emit_call()
349 self.emit_load_imm(mem, RAX, target as i64); in emit_call()
351 self.emit1(mem, 0xff); in emit_call()
352 self.emit1(mem, 0xd0); in emit_call()
355 fn emit_jump_offset(&mut self, mem: &mut JitMemory, target_pc: isize) { in emit_jump_offset()
357 offset_loc: mem.offset, in emit_jump_offset()
361 self.emit4(mem, 0); in emit_jump_offset()
364 fn emit_jcc(&mut self, mem: &mut JitMemory, code: u8, target_pc: isize) { in emit_jcc()
365 self.emit1(mem, 0x0f); in emit_jcc()
366 self.emit1(mem, code); in emit_jcc()
367 self.emit_jump_offset(mem, target_pc); in emit_jcc()
370 fn emit_jmp(&mut self, mem: &mut JitMemory, target_pc: isize) { in emit_jmp()
371 self.emit1(mem, 0xe9); in emit_jmp()
372 self.emit_jump_offset(mem, target_pc); in emit_jmp()
375 fn set_anchor(&mut self, mem: &mut JitMemory, target: isize) { in set_anchor()
376 self.special_targets.insert(target, mem.offset); in set_anchor()
381 mem: &mut JitMemory, in emit_muldivmod()
397 self.emit_alu32(mem, 0x31, dst, dst); in emit_muldivmod()
405 self.emit_load_imm(mem, RCX, pc as i64); in emit_muldivmod()
409 self.emit_alu64(mem, 0x85, src, src); in emit_muldivmod()
411 self.emit_alu32(mem, 0x85, src, src); in emit_muldivmod()
421 self.emit_direct_jcc(mem, 0x85, offset); in emit_muldivmod()
424 self.emit_alu32(mem, 0x31, dst, dst); in emit_muldivmod()
425 self.emit_jmp(mem, (pc + 1) as isize); in emit_muldivmod()
429 self.emit_jcc(mem, 0x84, (pc + 1) as isize); in emit_muldivmod()
434 self.emit_push(mem, RAX); in emit_muldivmod()
437 self.emit_push(mem, RDX); in emit_muldivmod()
440 self.emit_load_imm(mem, RCX, imm as i64); in emit_muldivmod()
442 self.emit_mov(mem, src, RCX); in emit_muldivmod()
445 self.emit_mov(mem, dst, RAX); in emit_muldivmod()
449 self.emit_alu32(mem, 0x31, RDX, RDX); in emit_muldivmod()
453 self.emit_rex(mem, 1, 0, 0, 0); in emit_muldivmod()
457 self.emit_alu32(mem, 0xf7, if mul { 4 } else { 6 }, RCX); in emit_muldivmod()
461 self.emit_mov(mem, RDX, dst); in emit_muldivmod()
463 self.emit_pop(mem, RDX); in emit_muldivmod()
467 self.emit_mov(mem, RAX, dst); in emit_muldivmod()
469 self.emit_pop(mem, RAX); in emit_muldivmod()
475 mem: &mut JitMemory, in jit_compile()
481 self.emit_push(mem, RBP); in jit_compile()
482 self.emit_push(mem, RBX); in jit_compile()
483 self.emit_push(mem, R13); in jit_compile()
484 self.emit_push(mem, R14); in jit_compile()
485 self.emit_push(mem, R15); in jit_compile()
495 self.emit_mov(mem, RDX, R10); in jit_compile()
501 self.emit_mov(mem, RDX, map_register(1)); in jit_compile()
507 self.emit_mov(mem, RDI, map_register(1)); in jit_compile()
513 self.emit_alu64(mem, 0x01, RDI, R8); // add mbuff to mem_offset in R8 in jit_compile()
514 self.emit_store(mem, OperandSize::S64, RDX, R8, 0); // set mem at mbuff + mem_offset in jit_compile()
516 self.emit_load(mem, OperandSize::S64, RDX, R8, 0); // load mem into R8 in jit_compile()
517 self.emit_alu64(mem, 0x01, RCX, R8); // add mem_len to mem (= mem_end) in jit_compile()
518 self.emit_alu64(mem, 0x01, RDI, R9); // add mbuff to mem_end_offset in jit_compile()
519 self.emit_store(mem, OperandSize::S64, R8, R9, 0); // store mem_end in jit_compile()
523 self.emit_mov(mem, RDI, map_register(1)); in jit_compile()
529 self.emit_mov(mem, RSP, map_register(10)); in jit_compile()
532 self.emit_alu64_imm32(mem, 0x81, 5, RSP, ebpf::STACK_SIZE as i32); in jit_compile()
540 self.pc_locs[insn_ptr] = mem.offset; in jit_compile()
549 ebpf::LD_ABS_B => self.emit_load(mem, OperandSize::S8, R10, RAX, insn.imm), in jit_compile()
550 ebpf::LD_ABS_H => self.emit_load(mem, OperandSize::S16, R10, RAX, insn.imm), in jit_compile()
551 ebpf::LD_ABS_W => self.emit_load(mem, OperandSize::S32, R10, RAX, insn.imm), in jit_compile()
552 ebpf::LD_ABS_DW => self.emit_load(mem, OperandSize::S64, R10, RAX, insn.imm), in jit_compile()
554 self.emit_mov(mem, R10, R11); // load mem into R11 in jit_compile()
555 self.emit_alu64(mem, 0x01, src, R11); // add src to R11 in jit_compile()
556 self.emit_load(mem, OperandSize::S8, R11, RAX, insn.imm); // ld R0, mem[src+imm] in jit_compile()
559 self.emit_mov(mem, R10, R11); // load mem into R11 in jit_compile()
560 self.emit_alu64(mem, 0x01, src, R11); // add src to R11 in jit_compile()
561 … self.emit_load(mem, OperandSize::S16, R11, RAX, insn.imm); // ld R0, mem[src+imm] in jit_compile()
564 self.emit_mov(mem, R10, R11); // load mem into R11 in jit_compile()
565 self.emit_alu64(mem, 0x01, src, R11); // add src to R11 in jit_compile()
566 … self.emit_load(mem, OperandSize::S32, R11, RAX, insn.imm); // ld R0, mem[src+imm] in jit_compile()
569 self.emit_mov(mem, R10, R11); // load mem into R11 in jit_compile()
570 self.emit_alu64(mem, 0x01, src, R11); // add src to R11 in jit_compile()
571 … self.emit_load(mem, OperandSize::S64, R11, RAX, insn.imm); // ld R0, mem[src+imm] in jit_compile()
578 self.emit_load_imm(mem, dst, imm as i64); in jit_compile()
582 ebpf::LD_B_REG => self.emit_load(mem, OperandSize::S8, src, dst, insn.off as i32), in jit_compile()
583 ebpf::LD_H_REG => self.emit_load(mem, OperandSize::S16, src, dst, insn.off as i32), in jit_compile()
584 ebpf::LD_W_REG => self.emit_load(mem, OperandSize::S32, src, dst, insn.off as i32), in jit_compile()
585 ebpf::LD_DW_REG => self.emit_load(mem, OperandSize::S64, src, dst, insn.off as i32), in jit_compile()
589 self.emit_store_imm32(mem, OperandSize::S8, dst, insn.off as i32, insn.imm) in jit_compile()
592 self.emit_store_imm32(mem, OperandSize::S16, dst, insn.off as i32, insn.imm) in jit_compile()
595 self.emit_store_imm32(mem, OperandSize::S32, dst, insn.off as i32, insn.imm) in jit_compile()
598 self.emit_store_imm32(mem, OperandSize::S64, dst, insn.off as i32, insn.imm) in jit_compile()
602 ebpf::ST_B_REG => self.emit_store(mem, OperandSize::S8, src, dst, insn.off as i32), in jit_compile()
603 ebpf::ST_H_REG => self.emit_store(mem, OperandSize::S16, src, dst, insn.off as i32), in jit_compile()
604 ebpf::ST_W_REG => self.emit_store(mem, OperandSize::S32, src, dst, insn.off as i32), in jit_compile()
606 self.emit_store(mem, OperandSize::S64, src, dst, insn.off as i32) in jit_compile()
612 ebpf::ADD32_IMM => self.emit_alu32_imm32(mem, 0x81, 0, dst, insn.imm), in jit_compile()
613 ebpf::ADD32_REG => self.emit_alu32(mem, 0x01, src, dst), in jit_compile()
614 ebpf::SUB32_IMM => self.emit_alu32_imm32(mem, 0x81, 5, dst, insn.imm), in jit_compile()
615 ebpf::SUB32_REG => self.emit_alu32(mem, 0x29, src, dst), in jit_compile()
622 self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) in jit_compile()
624 ebpf::OR32_IMM => self.emit_alu32_imm32(mem, 0x81, 1, dst, insn.imm), in jit_compile()
625 ebpf::OR32_REG => self.emit_alu32(mem, 0x09, src, dst), in jit_compile()
626 ebpf::AND32_IMM => self.emit_alu32_imm32(mem, 0x81, 4, dst, insn.imm), in jit_compile()
627 ebpf::AND32_REG => self.emit_alu32(mem, 0x21, src, dst), in jit_compile()
628 ebpf::LSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 4, dst, insn.imm as i8), in jit_compile()
630 self.emit_mov(mem, src, RCX); in jit_compile()
631 self.emit_alu32(mem, 0xd3, 4, dst); in jit_compile()
633 ebpf::RSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 5, dst, insn.imm as i8), in jit_compile()
635 self.emit_mov(mem, src, RCX); in jit_compile()
636 self.emit_alu32(mem, 0xd3, 5, dst); in jit_compile()
638 ebpf::NEG32 => self.emit_alu32(mem, 0xf7, 3, dst), in jit_compile()
639 ebpf::XOR32_IMM => self.emit_alu32_imm32(mem, 0x81, 6, dst, insn.imm), in jit_compile()
640 ebpf::XOR32_REG => self.emit_alu32(mem, 0x31, src, dst), in jit_compile()
641 ebpf::MOV32_IMM => self.emit_alu32_imm32(mem, 0xc7, 0, dst, insn.imm), in jit_compile()
642 ebpf::MOV32_REG => self.emit_mov(mem, src, dst), in jit_compile()
643 ebpf::ARSH32_IMM => self.emit_alu32_imm8(mem, 0xc1, 7, dst, insn.imm as i8), in jit_compile()
645 self.emit_mov(mem, src, RCX); in jit_compile()
646 self.emit_alu32(mem, 0xd3, 7, dst); in jit_compile()
653 self.emit1(mem, 0x66); // 16-bit override in jit_compile()
654 self.emit_alu32_imm8(mem, 0xc1, 0, dst, 8); in jit_compile()
656 self.emit_alu32_imm32(mem, 0x81, 4, dst, 0xffff); in jit_compile()
664 self.emit_basic_rex(mem, bit, 0, dst); in jit_compile()
665 self.emit1(mem, 0x0f); in jit_compile()
666 self.emit1(mem, 0xc8 | (dst & 0b111)); in jit_compile()
673 ebpf::ADD64_IMM => self.emit_alu64_imm32(mem, 0x81, 0, dst, insn.imm), in jit_compile()
674 ebpf::ADD64_REG => self.emit_alu64(mem, 0x01, src, dst), in jit_compile()
675 ebpf::SUB64_IMM => self.emit_alu64_imm32(mem, 0x81, 5, dst, insn.imm), in jit_compile()
676 ebpf::SUB64_REG => self.emit_alu64(mem, 0x29, src, dst), in jit_compile()
683 self.emit_muldivmod(mem, insn_ptr as u16, insn.opc, src, dst, insn.imm) in jit_compile()
685 ebpf::OR64_IMM => self.emit_alu64_imm32(mem, 0x81, 1, dst, insn.imm), in jit_compile()
686 ebpf::OR64_REG => self.emit_alu64(mem, 0x09, src, dst), in jit_compile()
687 ebpf::AND64_IMM => self.emit_alu64_imm32(mem, 0x81, 4, dst, insn.imm), in jit_compile()
688 ebpf::AND64_REG => self.emit_alu64(mem, 0x21, src, dst), in jit_compile()
689 ebpf::LSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 4, dst, insn.imm as i8), in jit_compile()
691 self.emit_mov(mem, src, RCX); in jit_compile()
692 self.emit_alu64(mem, 0xd3, 4, dst); in jit_compile()
694 ebpf::RSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 5, dst, insn.imm as i8), in jit_compile()
696 self.emit_mov(mem, src, RCX); in jit_compile()
697 self.emit_alu64(mem, 0xd3, 5, dst); in jit_compile()
699 ebpf::NEG64 => self.emit_alu64(mem, 0xf7, 3, dst), in jit_compile()
700 ebpf::XOR64_IMM => self.emit_alu64_imm32(mem, 0x81, 6, dst, insn.imm), in jit_compile()
701 ebpf::XOR64_REG => self.emit_alu64(mem, 0x31, src, dst), in jit_compile()
702 ebpf::MOV64_IMM => self.emit_load_imm(mem, dst, insn.imm as i64), in jit_compile()
703 ebpf::MOV64_REG => self.emit_mov(mem, src, dst), in jit_compile()
704 ebpf::ARSH64_IMM => self.emit_alu64_imm8(mem, 0xc1, 7, dst, insn.imm as i8), in jit_compile()
706 self.emit_mov(mem, src, RCX); in jit_compile()
707 self.emit_alu64(mem, 0xd3, 7, dst); in jit_compile()
711 ebpf::JA => self.emit_jmp(mem, target_pc), in jit_compile()
713 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
714 self.emit_jcc(mem, 0x84, target_pc); in jit_compile()
717 self.emit_cmp(mem, src, dst); in jit_compile()
718 self.emit_jcc(mem, 0x84, target_pc); in jit_compile()
721 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
722 self.emit_jcc(mem, 0x87, target_pc); in jit_compile()
725 self.emit_cmp(mem, src, dst); in jit_compile()
726 self.emit_jcc(mem, 0x87, target_pc); in jit_compile()
729 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
730 self.emit_jcc(mem, 0x83, target_pc); in jit_compile()
733 self.emit_cmp(mem, src, dst); in jit_compile()
734 self.emit_jcc(mem, 0x83, target_pc); in jit_compile()
737 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
738 self.emit_jcc(mem, 0x82, target_pc); in jit_compile()
741 self.emit_cmp(mem, src, dst); in jit_compile()
742 self.emit_jcc(mem, 0x82, target_pc); in jit_compile()
745 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
746 self.emit_jcc(mem, 0x86, target_pc); in jit_compile()
749 self.emit_cmp(mem, src, dst); in jit_compile()
750 self.emit_jcc(mem, 0x86, target_pc); in jit_compile()
753 self.emit_alu64_imm32(mem, 0xf7, 0, dst, insn.imm); in jit_compile()
754 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
757 self.emit_alu64(mem, 0x85, src, dst); in jit_compile()
758 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
761 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
762 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
765 self.emit_cmp(mem, src, dst); in jit_compile()
766 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
769 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
770 self.emit_jcc(mem, 0x8f, target_pc); in jit_compile()
773 self.emit_cmp(mem, src, dst); in jit_compile()
774 self.emit_jcc(mem, 0x8f, target_pc); in jit_compile()
777 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
778 self.emit_jcc(mem, 0x8d, target_pc); in jit_compile()
781 self.emit_cmp(mem, src, dst); in jit_compile()
782 self.emit_jcc(mem, 0x8d, target_pc); in jit_compile()
785 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
786 self.emit_jcc(mem, 0x8c, target_pc); in jit_compile()
789 self.emit_cmp(mem, src, dst); in jit_compile()
790 self.emit_jcc(mem, 0x8c, target_pc); in jit_compile()
793 self.emit_cmp_imm32(mem, dst, insn.imm); in jit_compile()
794 self.emit_jcc(mem, 0x8e, target_pc); in jit_compile()
797 self.emit_cmp(mem, src, dst); in jit_compile()
798 self.emit_jcc(mem, 0x8e, target_pc); in jit_compile()
803 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
804 self.emit_jcc(mem, 0x84, target_pc); in jit_compile()
807 self.emit_cmp32(mem, src, dst); in jit_compile()
808 self.emit_jcc(mem, 0x84, target_pc); in jit_compile()
811 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
812 self.emit_jcc(mem, 0x87, target_pc); in jit_compile()
815 self.emit_cmp32(mem, src, dst); in jit_compile()
816 self.emit_jcc(mem, 0x87, target_pc); in jit_compile()
819 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
820 self.emit_jcc(mem, 0x83, target_pc); in jit_compile()
823 self.emit_cmp32(mem, src, dst); in jit_compile()
824 self.emit_jcc(mem, 0x83, target_pc); in jit_compile()
827 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
828 self.emit_jcc(mem, 0x82, target_pc); in jit_compile()
831 self.emit_cmp32(mem, src, dst); in jit_compile()
832 self.emit_jcc(mem, 0x82, target_pc); in jit_compile()
835 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
836 self.emit_jcc(mem, 0x86, target_pc); in jit_compile()
839 self.emit_cmp32(mem, src, dst); in jit_compile()
840 self.emit_jcc(mem, 0x86, target_pc); in jit_compile()
843 self.emit_alu32_imm32(mem, 0xf7, 0, dst, insn.imm); in jit_compile()
844 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
847 self.emit_alu32(mem, 0x85, src, dst); in jit_compile()
848 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
851 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
852 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
855 self.emit_cmp32(mem, src, dst); in jit_compile()
856 self.emit_jcc(mem, 0x85, target_pc); in jit_compile()
859 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
860 self.emit_jcc(mem, 0x8f, target_pc); in jit_compile()
863 self.emit_cmp32(mem, src, dst); in jit_compile()
864 self.emit_jcc(mem, 0x8f, target_pc); in jit_compile()
867 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
868 self.emit_jcc(mem, 0x8d, target_pc); in jit_compile()
871 self.emit_cmp32(mem, src, dst); in jit_compile()
872 self.emit_jcc(mem, 0x8d, target_pc); in jit_compile()
875 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
876 self.emit_jcc(mem, 0x8c, target_pc); in jit_compile()
879 self.emit_cmp32(mem, src, dst); in jit_compile()
880 self.emit_jcc(mem, 0x8c, target_pc); in jit_compile()
883 self.emit_cmp32_imm32(mem, dst, insn.imm); in jit_compile()
884 self.emit_jcc(mem, 0x8e, target_pc); in jit_compile()
887 self.emit_cmp32(mem, src, dst); in jit_compile()
888 self.emit_jcc(mem, 0x8e, target_pc); in jit_compile()
897 self.emit_mov(mem, R9, RCX); in jit_compile()
898 self.emit_call(mem, *helper as usize); in jit_compile()
914 self.emit_jmp(mem, TARGET_PC_EXIT); in jit_compile()
933 self.set_anchor(mem, TARGET_PC_EXIT); in jit_compile()
937 self.emit_mov(mem, map_register(0), RAX); in jit_compile()
941 self.emit_alu64_imm32(mem, 0x81, 0, RSP, ebpf::STACK_SIZE as i32); in jit_compile()
943 self.emit_pop(mem, R15); in jit_compile()
944 self.emit_pop(mem, R14); in jit_compile()
945 self.emit_pop(mem, R13); in jit_compile()
946 self.emit_pop(mem, RBX); in jit_compile()
947 self.emit_pop(mem, RBP); in jit_compile()
949 self.emit1(mem, 0xc3); // ret in jit_compile()
954 fn resolve_jumps(&mut self, mem: &mut JitMemory) -> Result<(), Error> { in resolve_jumps()
963 let offset_loc = jump.offset_loc as i32 + std::mem::size_of::<i32>() as i32; in resolve_jumps()
966 let offset_ptr = mem.contents.as_ptr().add(jump.offset_loc); in resolve_jumps()
971 std::mem::size_of::<i32>(), in resolve_jumps()
992 let mut raw: mem::MaybeUninit<*mut libc::c_void> = mem::MaybeUninit::uninit(); in new()
1007 let mut mem = JitMemory { in new() localVariable
1013 jit.jit_compile(&mut mem, prog, use_mbuff, update_data_ptr, helpers)?; in new()
1014 jit.resolve_jumps(&mut mem)?; in new()
1016 Ok(mem) in new()
1020 unsafe { mem::transmute(self.contents.as_ptr()) } in get_prog()