Lines Matching refs:dst_reg
2430 return insn->dst_reg; in insn_def_regno()
2437 int dst_reg = insn_def_regno(insn); in insn_has_def32() local
2439 if (dst_reg == -1) in insn_has_def32()
2442 return !is_reg64(env, insn, dst_reg, NULL, DST_OP); in insn_has_def32()
2574 u32 dreg = 1u << insn->dst_reg; in backtrack_insn()
2650 if (insn->dst_reg != BPF_REG_FP) in backtrack_insn()
3043 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg; in check_stack_write_fixed_off() local
3081 if (dst_reg != BPF_REG_FP) { in check_stack_write_fixed_off()
5009 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_atomic()
5032 if (is_ctx_reg(env, insn->dst_reg) || in check_atomic()
5033 is_pkt_reg(env, insn->dst_reg) || in check_atomic()
5034 is_flow_key_reg(env, insn->dst_reg) || in check_atomic()
5035 is_sk_reg(env, insn->dst_reg)) { in check_atomic()
5037 insn->dst_reg, in check_atomic()
5038 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in check_atomic()
5062 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5065 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
5072 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, in check_atomic()
7992 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
7994 mark_reg_unknown(env, regs, insn->dst_reg); in sanitize_speculative_path()
8005 struct bpf_reg_state *dst_reg, in sanitize_ptr_alu() argument
8013 bool ptr_is_dst_reg = ptr_reg == dst_reg; in sanitize_ptr_alu()
8087 tmp = *dst_reg; in sanitize_ptr_alu()
8088 *dst_reg = *ptr_reg; in sanitize_ptr_alu()
8093 *dst_reg = tmp; in sanitize_ptr_alu()
8113 const struct bpf_reg_state *dst_reg) in sanitize_err() argument
8117 u32 dst = insn->dst_reg, src = insn->src_reg; in sanitize_err()
8122 off_reg == dst_reg ? dst : src, err); in sanitize_err()
8126 off_reg == dst_reg ? src : dst, err); in sanitize_err()
8185 const struct bpf_reg_state *dst_reg) in sanitize_check_bounds() argument
8187 u32 dst = insn->dst_reg; in sanitize_check_bounds()
8195 switch (dst_reg->type) { in sanitize_check_bounds()
8197 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, in sanitize_check_bounds()
8198 dst_reg->off + dst_reg->var_off.value)) in sanitize_check_bounds()
8202 if (check_map_access(env, dst, dst_reg->off, 1, false, ACCESS_HELPER)) { in sanitize_check_bounds()
8227 struct bpf_reg_state *regs = state->regs, *dst_reg; in adjust_ptr_min_max_vals() local
8235 u32 dst = insn->dst_reg; in adjust_ptr_min_max_vals()
8238 dst_reg = ®s[dst]; in adjust_ptr_min_max_vals()
8245 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8252 __mark_reg_unknown(env, dst_reg); in adjust_ptr_min_max_vals()
8289 dst_reg->type = ptr_reg->type; in adjust_ptr_min_max_vals()
8290 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
8297 __mark_reg32_unbounded(dst_reg); in adjust_ptr_min_max_vals()
8300 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8303 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8314 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
8315 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
8316 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
8317 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
8318 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
8319 dst_reg->off = ptr_reg->off + smin_val; in adjust_ptr_min_max_vals()
8320 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8334 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
8335 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
8337 dst_reg->smin_value = smin_ptr + smin_val; in adjust_ptr_min_max_vals()
8338 dst_reg->smax_value = smax_ptr + smax_val; in adjust_ptr_min_max_vals()
8342 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
8343 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
8345 dst_reg->umin_value = umin_ptr + umin_val; in adjust_ptr_min_max_vals()
8346 dst_reg->umax_value = umax_ptr + umax_val; in adjust_ptr_min_max_vals()
8348 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
8349 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
8350 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8352 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8354 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
8358 if (dst_reg == off_reg) { in adjust_ptr_min_max_vals()
8376 dst_reg->smin_value = smin_ptr; in adjust_ptr_min_max_vals()
8377 dst_reg->smax_value = smax_ptr; in adjust_ptr_min_max_vals()
8378 dst_reg->umin_value = umin_ptr; in adjust_ptr_min_max_vals()
8379 dst_reg->umax_value = umax_ptr; in adjust_ptr_min_max_vals()
8380 dst_reg->var_off = ptr_reg->var_off; in adjust_ptr_min_max_vals()
8381 dst_reg->id = ptr_reg->id; in adjust_ptr_min_max_vals()
8382 dst_reg->off = ptr_reg->off - smin_val; in adjust_ptr_min_max_vals()
8383 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8392 dst_reg->smin_value = S64_MIN; in adjust_ptr_min_max_vals()
8393 dst_reg->smax_value = S64_MAX; in adjust_ptr_min_max_vals()
8395 dst_reg->smin_value = smin_ptr - smax_val; in adjust_ptr_min_max_vals()
8396 dst_reg->smax_value = smax_ptr - smin_val; in adjust_ptr_min_max_vals()
8400 dst_reg->umin_value = 0; in adjust_ptr_min_max_vals()
8401 dst_reg->umax_value = U64_MAX; in adjust_ptr_min_max_vals()
8404 dst_reg->umin_value = umin_ptr - umax_val; in adjust_ptr_min_max_vals()
8405 dst_reg->umax_value = umax_ptr - umin_val; in adjust_ptr_min_max_vals()
8407 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); in adjust_ptr_min_max_vals()
8408 dst_reg->off = ptr_reg->off; in adjust_ptr_min_max_vals()
8409 dst_reg->raw = ptr_reg->raw; in adjust_ptr_min_max_vals()
8411 dst_reg->id = ++env->id_gen; in adjust_ptr_min_max_vals()
8414 memset(&dst_reg->raw, 0, sizeof(dst_reg->raw)); in adjust_ptr_min_max_vals()
8431 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) in adjust_ptr_min_max_vals()
8433 reg_bounds_sync(dst_reg); in adjust_ptr_min_max_vals()
8434 if (sanitize_check_bounds(env, insn, dst_reg) < 0) in adjust_ptr_min_max_vals()
8437 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, in adjust_ptr_min_max_vals()
8440 return sanitize_err(env, insn, ret, off_reg, dst_reg); in adjust_ptr_min_max_vals()
8446 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg, in scalar32_min_max_add() argument
8454 if (signed_add32_overflows(dst_reg->s32_min_value, smin_val) || in scalar32_min_max_add()
8455 signed_add32_overflows(dst_reg->s32_max_value, smax_val)) { in scalar32_min_max_add()
8456 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_add()
8457 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_add()
8459 dst_reg->s32_min_value += smin_val; in scalar32_min_max_add()
8460 dst_reg->s32_max_value += smax_val; in scalar32_min_max_add()
8462 if (dst_reg->u32_min_value + umin_val < umin_val || in scalar32_min_max_add()
8463 dst_reg->u32_max_value + umax_val < umax_val) { in scalar32_min_max_add()
8464 dst_reg->u32_min_value = 0; in scalar32_min_max_add()
8465 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_add()
8467 dst_reg->u32_min_value += umin_val; in scalar32_min_max_add()
8468 dst_reg->u32_max_value += umax_val; in scalar32_min_max_add()
8472 static void scalar_min_max_add(struct bpf_reg_state *dst_reg, in scalar_min_max_add() argument
8480 if (signed_add_overflows(dst_reg->smin_value, smin_val) || in scalar_min_max_add()
8481 signed_add_overflows(dst_reg->smax_value, smax_val)) { in scalar_min_max_add()
8482 dst_reg->smin_value = S64_MIN; in scalar_min_max_add()
8483 dst_reg->smax_value = S64_MAX; in scalar_min_max_add()
8485 dst_reg->smin_value += smin_val; in scalar_min_max_add()
8486 dst_reg->smax_value += smax_val; in scalar_min_max_add()
8488 if (dst_reg->umin_value + umin_val < umin_val || in scalar_min_max_add()
8489 dst_reg->umax_value + umax_val < umax_val) { in scalar_min_max_add()
8490 dst_reg->umin_value = 0; in scalar_min_max_add()
8491 dst_reg->umax_value = U64_MAX; in scalar_min_max_add()
8493 dst_reg->umin_value += umin_val; in scalar_min_max_add()
8494 dst_reg->umax_value += umax_val; in scalar_min_max_add()
8498 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg, in scalar32_min_max_sub() argument
8506 if (signed_sub32_overflows(dst_reg->s32_min_value, smax_val) || in scalar32_min_max_sub()
8507 signed_sub32_overflows(dst_reg->s32_max_value, smin_val)) { in scalar32_min_max_sub()
8509 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_sub()
8510 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_sub()
8512 dst_reg->s32_min_value -= smax_val; in scalar32_min_max_sub()
8513 dst_reg->s32_max_value -= smin_val; in scalar32_min_max_sub()
8515 if (dst_reg->u32_min_value < umax_val) { in scalar32_min_max_sub()
8517 dst_reg->u32_min_value = 0; in scalar32_min_max_sub()
8518 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_sub()
8521 dst_reg->u32_min_value -= umax_val; in scalar32_min_max_sub()
8522 dst_reg->u32_max_value -= umin_val; in scalar32_min_max_sub()
8526 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg, in scalar_min_max_sub() argument
8534 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || in scalar_min_max_sub()
8535 signed_sub_overflows(dst_reg->smax_value, smin_val)) { in scalar_min_max_sub()
8537 dst_reg->smin_value = S64_MIN; in scalar_min_max_sub()
8538 dst_reg->smax_value = S64_MAX; in scalar_min_max_sub()
8540 dst_reg->smin_value -= smax_val; in scalar_min_max_sub()
8541 dst_reg->smax_value -= smin_val; in scalar_min_max_sub()
8543 if (dst_reg->umin_value < umax_val) { in scalar_min_max_sub()
8545 dst_reg->umin_value = 0; in scalar_min_max_sub()
8546 dst_reg->umax_value = U64_MAX; in scalar_min_max_sub()
8549 dst_reg->umin_value -= umax_val; in scalar_min_max_sub()
8550 dst_reg->umax_value -= umin_val; in scalar_min_max_sub()
8554 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg, in scalar32_min_max_mul() argument
8561 if (smin_val < 0 || dst_reg->s32_min_value < 0) { in scalar32_min_max_mul()
8563 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
8569 if (umax_val > U16_MAX || dst_reg->u32_max_value > U16_MAX) { in scalar32_min_max_mul()
8571 __mark_reg32_unbounded(dst_reg); in scalar32_min_max_mul()
8574 dst_reg->u32_min_value *= umin_val; in scalar32_min_max_mul()
8575 dst_reg->u32_max_value *= umax_val; in scalar32_min_max_mul()
8576 if (dst_reg->u32_max_value > S32_MAX) { in scalar32_min_max_mul()
8578 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_mul()
8579 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_mul()
8581 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_mul()
8582 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_mul()
8586 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg, in scalar_min_max_mul() argument
8593 if (smin_val < 0 || dst_reg->smin_value < 0) { in scalar_min_max_mul()
8595 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
8601 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { in scalar_min_max_mul()
8603 __mark_reg64_unbounded(dst_reg); in scalar_min_max_mul()
8606 dst_reg->umin_value *= umin_val; in scalar_min_max_mul()
8607 dst_reg->umax_value *= umax_val; in scalar_min_max_mul()
8608 if (dst_reg->umax_value > S64_MAX) { in scalar_min_max_mul()
8610 dst_reg->smin_value = S64_MIN; in scalar_min_max_mul()
8611 dst_reg->smax_value = S64_MAX; in scalar_min_max_mul()
8613 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_mul()
8614 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_mul()
8618 static void scalar32_min_max_and(struct bpf_reg_state *dst_reg, in scalar32_min_max_and() argument
8622 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_and()
8623 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_and()
8628 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_and()
8635 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_and()
8636 dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val); in scalar32_min_max_and()
8637 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_and()
8641 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_and()
8642 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_and()
8647 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_and()
8648 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_and()
8652 static void scalar_min_max_and(struct bpf_reg_state *dst_reg, in scalar_min_max_and() argument
8656 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_and()
8661 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_and()
8668 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_and()
8669 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); in scalar_min_max_and()
8670 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_and()
8674 dst_reg->smin_value = S64_MIN; in scalar_min_max_and()
8675 dst_reg->smax_value = S64_MAX; in scalar_min_max_and()
8680 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_and()
8681 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_and()
8684 __update_reg_bounds(dst_reg); in scalar_min_max_and()
8687 static void scalar32_min_max_or(struct bpf_reg_state *dst_reg, in scalar32_min_max_or() argument
8691 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_or()
8692 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_or()
8697 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_or()
8704 dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val); in scalar32_min_max_or()
8705 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_or()
8706 if (dst_reg->s32_min_value < 0 || smin_val < 0) { in scalar32_min_max_or()
8710 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_or()
8711 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_or()
8716 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_or()
8717 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_or()
8721 static void scalar_min_max_or(struct bpf_reg_state *dst_reg, in scalar_min_max_or() argument
8725 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_or()
8730 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_or()
8737 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); in scalar_min_max_or()
8738 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_or()
8739 if (dst_reg->smin_value < 0 || smin_val < 0) { in scalar_min_max_or()
8743 dst_reg->smin_value = S64_MIN; in scalar_min_max_or()
8744 dst_reg->smax_value = S64_MAX; in scalar_min_max_or()
8749 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_or()
8750 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_or()
8753 __update_reg_bounds(dst_reg); in scalar_min_max_or()
8756 static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg, in scalar32_min_max_xor() argument
8760 bool dst_known = tnum_subreg_is_const(dst_reg->var_off); in scalar32_min_max_xor()
8761 struct tnum var32_off = tnum_subreg(dst_reg->var_off); in scalar32_min_max_xor()
8765 __mark_reg32_known(dst_reg, var32_off.value); in scalar32_min_max_xor()
8770 dst_reg->u32_min_value = var32_off.value; in scalar32_min_max_xor()
8771 dst_reg->u32_max_value = var32_off.value | var32_off.mask; in scalar32_min_max_xor()
8773 if (dst_reg->s32_min_value >= 0 && smin_val >= 0) { in scalar32_min_max_xor()
8777 dst_reg->s32_min_value = dst_reg->u32_min_value; in scalar32_min_max_xor()
8778 dst_reg->s32_max_value = dst_reg->u32_max_value; in scalar32_min_max_xor()
8780 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_xor()
8781 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_xor()
8785 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg, in scalar_min_max_xor() argument
8789 bool dst_known = tnum_is_const(dst_reg->var_off); in scalar_min_max_xor()
8794 __mark_reg_known(dst_reg, dst_reg->var_off.value); in scalar_min_max_xor()
8799 dst_reg->umin_value = dst_reg->var_off.value; in scalar_min_max_xor()
8800 dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask; in scalar_min_max_xor()
8802 if (dst_reg->smin_value >= 0 && smin_val >= 0) { in scalar_min_max_xor()
8806 dst_reg->smin_value = dst_reg->umin_value; in scalar_min_max_xor()
8807 dst_reg->smax_value = dst_reg->umax_value; in scalar_min_max_xor()
8809 dst_reg->smin_value = S64_MIN; in scalar_min_max_xor()
8810 dst_reg->smax_value = S64_MAX; in scalar_min_max_xor()
8813 __update_reg_bounds(dst_reg); in scalar_min_max_xor()
8816 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar32_min_max_lsh() argument
8822 dst_reg->s32_min_value = S32_MIN; in __scalar32_min_max_lsh()
8823 dst_reg->s32_max_value = S32_MAX; in __scalar32_min_max_lsh()
8825 if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) { in __scalar32_min_max_lsh()
8826 dst_reg->u32_min_value = 0; in __scalar32_min_max_lsh()
8827 dst_reg->u32_max_value = U32_MAX; in __scalar32_min_max_lsh()
8829 dst_reg->u32_min_value <<= umin_val; in __scalar32_min_max_lsh()
8830 dst_reg->u32_max_value <<= umax_val; in __scalar32_min_max_lsh()
8834 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_lsh() argument
8840 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_lsh()
8842 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar32_min_max_lsh()
8843 dst_reg->var_off = tnum_subreg(tnum_lshift(subreg, umin_val)); in scalar32_min_max_lsh()
8848 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_lsh()
8849 __update_reg32_bounds(dst_reg); in scalar32_min_max_lsh()
8852 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg, in __scalar64_min_max_lsh() argument
8862 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_max_value >= 0) in __scalar64_min_max_lsh()
8863 dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32; in __scalar64_min_max_lsh()
8865 dst_reg->smax_value = S64_MAX; in __scalar64_min_max_lsh()
8867 if (umin_val == 32 && umax_val == 32 && dst_reg->s32_min_value >= 0) in __scalar64_min_max_lsh()
8868 dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32; in __scalar64_min_max_lsh()
8870 dst_reg->smin_value = S64_MIN; in __scalar64_min_max_lsh()
8873 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { in __scalar64_min_max_lsh()
8874 dst_reg->umin_value = 0; in __scalar64_min_max_lsh()
8875 dst_reg->umax_value = U64_MAX; in __scalar64_min_max_lsh()
8877 dst_reg->umin_value <<= umin_val; in __scalar64_min_max_lsh()
8878 dst_reg->umax_value <<= umax_val; in __scalar64_min_max_lsh()
8882 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg, in scalar_min_max_lsh() argument
8889 __scalar64_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
8890 __scalar32_min_max_lsh(dst_reg, umin_val, umax_val); in scalar_min_max_lsh()
8892 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); in scalar_min_max_lsh()
8894 __update_reg_bounds(dst_reg); in scalar_min_max_lsh()
8897 static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_rsh() argument
8900 struct tnum subreg = tnum_subreg(dst_reg->var_off); in scalar32_min_max_rsh()
8918 dst_reg->s32_min_value = S32_MIN; in scalar32_min_max_rsh()
8919 dst_reg->s32_max_value = S32_MAX; in scalar32_min_max_rsh()
8921 dst_reg->var_off = tnum_rshift(subreg, umin_val); in scalar32_min_max_rsh()
8922 dst_reg->u32_min_value >>= umax_val; in scalar32_min_max_rsh()
8923 dst_reg->u32_max_value >>= umin_val; in scalar32_min_max_rsh()
8925 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_rsh()
8926 __update_reg32_bounds(dst_reg); in scalar32_min_max_rsh()
8929 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg, in scalar_min_max_rsh() argument
8949 dst_reg->smin_value = S64_MIN; in scalar_min_max_rsh()
8950 dst_reg->smax_value = S64_MAX; in scalar_min_max_rsh()
8951 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); in scalar_min_max_rsh()
8952 dst_reg->umin_value >>= umax_val; in scalar_min_max_rsh()
8953 dst_reg->umax_value >>= umin_val; in scalar_min_max_rsh()
8959 __mark_reg32_unbounded(dst_reg); in scalar_min_max_rsh()
8960 __update_reg_bounds(dst_reg); in scalar_min_max_rsh()
8963 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar32_min_max_arsh() argument
8971 dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val); in scalar32_min_max_arsh()
8972 dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val); in scalar32_min_max_arsh()
8974 dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32); in scalar32_min_max_arsh()
8979 dst_reg->u32_min_value = 0; in scalar32_min_max_arsh()
8980 dst_reg->u32_max_value = U32_MAX; in scalar32_min_max_arsh()
8982 __mark_reg64_unbounded(dst_reg); in scalar32_min_max_arsh()
8983 __update_reg32_bounds(dst_reg); in scalar32_min_max_arsh()
8986 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg, in scalar_min_max_arsh() argument
8994 dst_reg->smin_value >>= umin_val; in scalar_min_max_arsh()
8995 dst_reg->smax_value >>= umin_val; in scalar_min_max_arsh()
8997 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64); in scalar_min_max_arsh()
9002 dst_reg->umin_value = 0; in scalar_min_max_arsh()
9003 dst_reg->umax_value = U64_MAX; in scalar_min_max_arsh()
9009 __mark_reg32_unbounded(dst_reg); in scalar_min_max_arsh()
9010 __update_reg_bounds(dst_reg); in scalar_min_max_arsh()
9019 struct bpf_reg_state *dst_reg, in adjust_scalar_min_max_vals() argument
9051 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9062 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9069 __mark_reg_unknown(env, dst_reg); in adjust_scalar_min_max_vals()
9095 scalar32_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9096 scalar_min_max_add(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9097 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9100 scalar32_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9101 scalar_min_max_sub(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9102 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9105 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9106 scalar32_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9107 scalar_min_max_mul(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9110 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9111 scalar32_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9112 scalar_min_max_and(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9115 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9116 scalar32_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9117 scalar_min_max_or(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9120 dst_reg->var_off = tnum_xor(dst_reg->var_off, src_reg.var_off); in adjust_scalar_min_max_vals()
9121 scalar32_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9122 scalar_min_max_xor(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9129 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9133 scalar32_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9135 scalar_min_max_lsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9142 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9146 scalar32_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9148 scalar_min_max_rsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9155 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9159 scalar32_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9161 scalar_min_max_arsh(dst_reg, &src_reg); in adjust_scalar_min_max_vals()
9164 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_scalar_min_max_vals()
9170 zext_32_to_64(dst_reg); in adjust_scalar_min_max_vals()
9171 reg_bounds_sync(dst_reg); in adjust_scalar_min_max_vals()
9183 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; in adjust_reg_min_max_vals() local
9188 dst_reg = ®s[insn->dst_reg]; in adjust_reg_min_max_vals()
9190 if (dst_reg->type != SCALAR_VALUE) in adjust_reg_min_max_vals()
9191 ptr_reg = dst_reg; in adjust_reg_min_max_vals()
9196 dst_reg->id = 0; in adjust_reg_min_max_vals()
9200 if (dst_reg->type != SCALAR_VALUE) { in adjust_reg_min_max_vals()
9206 mark_reg_unknown(env, regs, insn->dst_reg); in adjust_reg_min_max_vals()
9210 insn->dst_reg, in adjust_reg_min_max_vals()
9218 err = mark_chain_precision(env, insn->dst_reg); in adjust_reg_min_max_vals()
9222 src_reg, dst_reg); in adjust_reg_min_max_vals()
9230 dst_reg, src_reg); in adjust_reg_min_max_vals()
9231 } else if (dst_reg->precise) { in adjust_reg_min_max_vals()
9260 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); in adjust_reg_min_max_vals()
9288 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9292 if (is_pointer_value(env, insn->dst_reg)) { in check_alu_op()
9294 insn->dst_reg); in check_alu_op()
9299 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_alu_op()
9323 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9329 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; in check_alu_op() local
9341 *dst_reg = *src_reg; in check_alu_op()
9342 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
9343 dst_reg->subreg_def = DEF_NOT_SUBREG; in check_alu_op()
9352 *dst_reg = *src_reg; in check_alu_op()
9357 dst_reg->id = 0; in check_alu_op()
9358 dst_reg->live |= REG_LIVE_WRITTEN; in check_alu_op()
9359 dst_reg->subreg_def = env->insn_idx + 1; in check_alu_op()
9362 insn->dst_reg); in check_alu_op()
9364 zext_32_to_64(dst_reg); in check_alu_op()
9365 reg_bounds_sync(dst_reg); in check_alu_op()
9372 mark_reg_unknown(env, regs, insn->dst_reg); in check_alu_op()
9373 regs[insn->dst_reg].type = SCALAR_VALUE; in check_alu_op()
9375 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
9378 __mark_reg_known(regs + insn->dst_reg, in check_alu_op()
9406 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_alu_op()
9427 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in check_alu_op()
9438 struct bpf_reg_state *dst_reg, in find_good_pkt_pointers() argument
9446 if (dst_reg->off < 0 || in find_good_pkt_pointers()
9447 (dst_reg->off == 0 && range_right_open)) in find_good_pkt_pointers()
9451 if (dst_reg->umax_value > MAX_PACKET_OFF || in find_good_pkt_pointers()
9452 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) in find_good_pkt_pointers()
9458 new_range = dst_reg->off; in find_good_pkt_pointers()
9510 if (reg->type == type && reg->id == dst_reg->id) in find_good_pkt_pointers()
9718 static int is_pkt_ptr_branch_taken(struct bpf_reg_state *dst_reg, in is_pkt_ptr_branch_taken() argument
9725 pkt = dst_reg; in is_pkt_ptr_branch_taken()
9726 } else if (dst_reg->type == PTR_TO_PACKET_END) { in is_pkt_ptr_branch_taken()
9938 struct bpf_reg_state *dst_reg) in __reg_combine_min_max() argument
9940 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, in __reg_combine_min_max()
9941 dst_reg->umin_value); in __reg_combine_min_max()
9942 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, in __reg_combine_min_max()
9943 dst_reg->umax_value); in __reg_combine_min_max()
9944 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, in __reg_combine_min_max()
9945 dst_reg->smin_value); in __reg_combine_min_max()
9946 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, in __reg_combine_min_max()
9947 dst_reg->smax_value); in __reg_combine_min_max()
9948 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, in __reg_combine_min_max()
9949 dst_reg->var_off); in __reg_combine_min_max()
9951 reg_bounds_sync(dst_reg); in __reg_combine_min_max()
10036 struct bpf_reg_state *dst_reg, in try_match_pkt_pointers() argument
10050 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10052 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10055 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
10056 dst_reg->type, false); in try_match_pkt_pointers()
10057 mark_pkt_end(other_branch, insn->dst_reg, true); in try_match_pkt_pointers()
10058 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10060 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10071 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10073 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10076 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
10077 dst_reg->type, true); in try_match_pkt_pointers()
10078 mark_pkt_end(this_branch, insn->dst_reg, false); in try_match_pkt_pointers()
10079 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10081 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10092 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10094 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10097 find_good_pkt_pointers(this_branch, dst_reg, in try_match_pkt_pointers()
10098 dst_reg->type, true); in try_match_pkt_pointers()
10099 mark_pkt_end(other_branch, insn->dst_reg, false); in try_match_pkt_pointers()
10100 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10102 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10113 if ((dst_reg->type == PTR_TO_PACKET && in try_match_pkt_pointers()
10115 (dst_reg->type == PTR_TO_PACKET_META && in try_match_pkt_pointers()
10118 find_good_pkt_pointers(other_branch, dst_reg, in try_match_pkt_pointers()
10119 dst_reg->type, false); in try_match_pkt_pointers()
10120 mark_pkt_end(this_branch, insn->dst_reg, true); in try_match_pkt_pointers()
10121 } else if ((dst_reg->type == PTR_TO_PACKET_END && in try_match_pkt_pointers()
10123 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && in try_match_pkt_pointers()
10158 struct bpf_reg_state *dst_reg, *other_branch_regs, *src_reg = NULL; in check_cond_jmp_op() local
10195 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in check_cond_jmp_op()
10199 dst_reg = ®s[insn->dst_reg]; in check_cond_jmp_op()
10203 pred = is_branch_taken(dst_reg, insn->imm, opcode, is_jmp32); in check_cond_jmp_op()
10206 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
10212 pred = is_branch_taken(dst_reg, in check_cond_jmp_op()
10216 } else if (reg_is_pkt_pointer_any(dst_reg) && in check_cond_jmp_op()
10219 pred = is_pkt_ptr_branch_taken(dst_reg, src_reg, opcode); in check_cond_jmp_op()
10226 if (!__is_pointer_value(false, dst_reg)) in check_cond_jmp_op()
10227 err = mark_chain_precision(env, insn->dst_reg); in check_cond_jmp_op()
10275 if (dst_reg->type == SCALAR_VALUE && in check_cond_jmp_op()
10280 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10281 dst_reg, in check_cond_jmp_op()
10285 else if (tnum_is_const(dst_reg->var_off) || in check_cond_jmp_op()
10287 tnum_is_const(tnum_subreg(dst_reg->var_off)))) in check_cond_jmp_op()
10290 dst_reg->var_off.value, in check_cond_jmp_op()
10291 tnum_subreg(dst_reg->var_off).value, in check_cond_jmp_op()
10297 &other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10298 src_reg, dst_reg, opcode); in check_cond_jmp_op()
10306 } else if (dst_reg->type == SCALAR_VALUE) { in check_cond_jmp_op()
10307 reg_set_min_max(&other_branch_regs[insn->dst_reg], in check_cond_jmp_op()
10308 dst_reg, insn->imm, (u32)insn->imm, in check_cond_jmp_op()
10312 if (dst_reg->type == SCALAR_VALUE && dst_reg->id && in check_cond_jmp_op()
10313 !WARN_ON_ONCE(dst_reg->id != other_branch_regs[insn->dst_reg].id)) { in check_cond_jmp_op()
10314 find_equal_scalars(this_branch, dst_reg); in check_cond_jmp_op()
10315 find_equal_scalars(other_branch, &other_branch_regs[insn->dst_reg]); in check_cond_jmp_op()
10324 type_may_be_null(dst_reg->type)) { in check_cond_jmp_op()
10328 mark_ptr_or_null_regs(this_branch, insn->dst_reg, in check_cond_jmp_op()
10330 mark_ptr_or_null_regs(other_branch, insn->dst_reg, in check_cond_jmp_op()
10332 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], in check_cond_jmp_op()
10334 is_pointer_value(env, insn->dst_reg)) { in check_cond_jmp_op()
10336 insn->dst_reg); in check_cond_jmp_op()
10349 struct bpf_reg_state *dst_reg; in check_ld_imm() local
10362 err = check_reg_arg(env, insn->dst_reg, DST_OP); in check_ld_imm()
10366 dst_reg = ®s[insn->dst_reg]; in check_ld_imm()
10370 dst_reg->type = SCALAR_VALUE; in check_ld_imm()
10371 __mark_reg_known(®s[insn->dst_reg], imm); in check_ld_imm()
10379 mark_reg_known_zero(env, regs, insn->dst_reg); in check_ld_imm()
10382 dst_reg->type = aux->btf_var.reg_type; in check_ld_imm()
10383 switch (base_type(dst_reg->type)) { in check_ld_imm()
10385 dst_reg->mem_size = aux->btf_var.mem_size; in check_ld_imm()
10388 dst_reg->btf = aux->btf_var.btf; in check_ld_imm()
10389 dst_reg->btf_id = aux->btf_var.btf_id; in check_ld_imm()
10412 dst_reg->type = PTR_TO_FUNC; in check_ld_imm()
10413 dst_reg->subprogno = subprogno; in check_ld_imm()
10418 dst_reg->map_ptr = map; in check_ld_imm()
10422 dst_reg->type = PTR_TO_MAP_VALUE; in check_ld_imm()
10423 dst_reg->off = aux->map_off; in check_ld_imm()
10425 dst_reg->id = ++env->id_gen; in check_ld_imm()
10428 dst_reg->type = CONST_PTR_TO_MAP; in check_ld_imm()
10481 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || in check_ld_abs()
12289 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); in do_check()
12300 BPF_READ, insn->dst_reg, false); in do_check()
12346 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12350 dst_reg_type = regs[insn->dst_reg].type; in do_check()
12353 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12375 err = check_reg_arg(env, insn->dst_reg, SRC_OP); in do_check()
12379 if (is_ctx_reg(env, insn->dst_reg)) { in do_check()
12381 insn->dst_reg, in do_check()
12382 reg_type_str(env, reg_state(env, insn->dst_reg)->type)); in do_check()
12387 err = check_mem_access(env, env->insn_idx, insn->dst_reg, in do_check()
12404 insn->dst_reg != BPF_REG_0 || in do_check()
12428 insn->dst_reg != BPF_REG_0 || in do_check()
12441 insn->dst_reg != BPF_REG_0 || in do_check()
12802 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || in resolve_pseudo_ldimm64()
13394 rnd_hi32_patch[3].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
13423 zext_patch[1].dst_reg = load_reg; in opt_subreg_zext_lo32_rnd_hi32()
13602 insn->dst_reg, in convert_ctx_accesses()
13604 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
13609 insn->dst_reg, in convert_ctx_accesses()
13611 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, in convert_ctx_accesses()
13966 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), in do_misc_fixups()
13977 BPF_MOV32_REG(insn->dst_reg, insn->dst_reg), in do_misc_fixups()
14033 off_reg = issrc ? insn->src_reg : insn->dst_reg; in do_misc_fixups()
14047 *patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg); in do_misc_fixups()