Lines Matching refs:cqp
182 static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_arp_cache_entry() argument
189 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_arp_cache_entry()
199 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_arp_cache_entry()
207 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_arp_cache_entry()
219 static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_arp_cache_entry() argument
225 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_arp_cache_entry()
231 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_del_arp_cache_entry()
240 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_arp_cache_entry()
252 static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_apbvt_entry() argument
259 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_apbvt_entry()
267 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_apbvt_entry()
275 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_apbvt_entry()
300 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_qhash_table_entry() argument
310 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_qhash_table_entry()
351 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | in irdma_sc_manage_qhash_table_entry()
365 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_qhash_table_entry()
451 struct irdma_sc_cqp *cqp; in irdma_sc_qp_create() local
455 cqp = qp->dev->cqp; in irdma_sc_qp_create()
456 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || in irdma_sc_qp_create()
457 qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt) in irdma_sc_qp_create()
460 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_create()
479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_create()
487 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_create()
503 struct irdma_sc_cqp *cqp; in irdma_sc_qp_modify() local
508 cqp = qp->dev->cqp; in irdma_sc_qp_modify()
509 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_modify()
548 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_modify()
556 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_modify()
573 struct irdma_sc_cqp *cqp; in irdma_sc_qp_destroy() local
576 cqp = qp->dev->cqp; in irdma_sc_qp_destroy()
577 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_destroy()
589 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_destroy()
597 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_destroy()
761 static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_alloc_local_mac_entry() argument
767 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_local_mac_entry()
773 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_local_mac_entry()
784 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_local_mac_entry()
795 static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_local_mac_entry() argument
802 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_local_mac_entry()
811 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_local_mac_entry()
821 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_local_mac_entry()
833 static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_local_mac_entry() argument
840 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_local_mac_entry()
847 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_del_local_mac_entry()
859 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_local_mac_entry()
1060 struct irdma_sc_cqp *cqp; in irdma_sc_alloc_stag() local
1071 cqp = dev->cqp; in irdma_sc_alloc_stag()
1072 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_stag()
1096 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_stag()
1104 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_stag()
1122 struct irdma_sc_cqp *cqp; in irdma_sc_mr_reg_non_shared() local
1148 cqp = dev->cqp; in irdma_sc_mr_reg_non_shared()
1149 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mr_reg_non_shared()
1184 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mr_reg_non_shared()
1192 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mr_reg_non_shared()
1210 struct irdma_sc_cqp *cqp; in irdma_sc_dealloc_stag() local
1212 cqp = dev->cqp; in irdma_sc_dealloc_stag()
1213 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_dealloc_stag()
1224 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_dealloc_stag()
1232 irdma_sc_cqp_post_sq(cqp); in irdma_sc_dealloc_stag()
1249 struct irdma_sc_cqp *cqp; in irdma_sc_mw_alloc() local
1252 cqp = dev->cqp; in irdma_sc_mw_alloc()
1253 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mw_alloc()
1266 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mw_alloc()
1274 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mw_alloc()
2028 static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, in irdma_sc_gather_stats() argument
2038 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gather_stats()
2046 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_gather_stats()
2060 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gather_stats()
2061 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_gather_stats()
2063 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_gather_stats()
2075 static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, in irdma_sc_manage_stats_inst() argument
2082 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_stats_inst()
2088 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_manage_stats_inst()
2102 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_stats_inst()
2112 static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, in irdma_sc_set_up_map() argument
2119 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_set_up_map()
2131 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | in irdma_sc_set_up_map()
2142 irdma_sc_cqp_post_sq(cqp); in irdma_sc_set_up_map()
2154 static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, in irdma_sc_manage_ws_node() argument
2161 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_ws_node()
2169 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | in irdma_sc_manage_ws_node()
2184 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_ws_node()
2202 struct irdma_sc_cqp *cqp; in irdma_sc_qp_flush_wqes() local
2220 cqp = qp->pd->dev->cqp; in irdma_sc_qp_flush_wqes()
2221 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_flush_wqes()
2250 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_flush_wqes()
2258 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_flush_wqes()
2276 struct irdma_sc_cqp *cqp; in irdma_sc_gen_ae() local
2279 cqp = qp->pd->dev->cqp; in irdma_sc_gen_ae()
2280 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gen_ae()
2291 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_gen_ae()
2299 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gen_ae()
2315 struct irdma_sc_cqp *cqp; in irdma_sc_qp_upload_context() local
2318 cqp = dev->cqp; in irdma_sc_qp_upload_context()
2319 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_upload_context()
2330 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_upload_context()
2338 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_upload_context()
2350 static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, in irdma_sc_manage_push_page() argument
2358 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) in irdma_sc_manage_push_page()
2361 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_push_page()
2369 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_manage_push_page()
2378 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_push_page()
2389 static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, in irdma_sc_suspend_qp() argument
2395 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_suspend_qp()
2401 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_suspend_qp()
2408 irdma_sc_cqp_post_sq(cqp); in irdma_sc_suspend_qp()
2419 static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, in irdma_sc_resume_qp() argument
2425 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_resume_qp()
2434 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_resume_qp()
2441 irdma_sc_cqp_post_sq(cqp); in irdma_sc_resume_qp()
2501 struct irdma_sc_cqp *cqp; in irdma_sc_cq_create() local
2506 cqp = cq->dev->cqp; in irdma_sc_cq_create()
2507 if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt) in irdma_sc_cq_create()
2520 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_create()
2551 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_create()
2559 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_create()
2572 struct irdma_sc_cqp *cqp; in irdma_sc_cq_destroy() local
2577 cqp = cq->dev->cqp; in irdma_sc_cq_destroy()
2578 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_destroy()
2602 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_destroy()
2610 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_destroy()
2640 struct irdma_sc_cqp *cqp; in irdma_sc_cq_modify() local
2650 cqp = cq->dev->cqp; in irdma_sc_cq_modify()
2651 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_modify()
2676 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_modify()
2684 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_modify()
2713 static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, in irdma_get_cqp_reg_info() argument
2716 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_get_cqp_reg_info()
2727 static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail, in irdma_cqp_poll_registers() argument
2734 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error); in irdma_cqp_poll_registers()
2736 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_cqp_poll_registers()
2737 ibdev_dbg(to_ibdev(cqp->dev), in irdma_cqp_poll_registers()
2744 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_cqp_poll_registers()
2745 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS]++; in irdma_cqp_poll_registers()
2748 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_cqp_poll_registers()
3069 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, in irdma_sc_cqp_init() argument
3081 cqp->size = sizeof(*cqp); in irdma_sc_cqp_init()
3082 cqp->sq_size = info->sq_size; in irdma_sc_cqp_init()
3083 cqp->hw_sq_size = hw_sq_size; in irdma_sc_cqp_init()
3084 cqp->sq_base = info->sq; in irdma_sc_cqp_init()
3085 cqp->host_ctx = info->host_ctx; in irdma_sc_cqp_init()
3086 cqp->sq_pa = info->sq_pa; in irdma_sc_cqp_init()
3087 cqp->host_ctx_pa = info->host_ctx_pa; in irdma_sc_cqp_init()
3088 cqp->dev = info->dev; in irdma_sc_cqp_init()
3089 cqp->struct_ver = info->struct_ver; in irdma_sc_cqp_init()
3090 cqp->hw_maj_ver = info->hw_maj_ver; in irdma_sc_cqp_init()
3091 cqp->hw_min_ver = info->hw_min_ver; in irdma_sc_cqp_init()
3092 cqp->scratch_array = info->scratch_array; in irdma_sc_cqp_init()
3093 cqp->polarity = 0; in irdma_sc_cqp_init()
3094 cqp->en_datacenter_tcp = info->en_datacenter_tcp; in irdma_sc_cqp_init()
3095 cqp->ena_vf_count = info->ena_vf_count; in irdma_sc_cqp_init()
3096 cqp->hmc_profile = info->hmc_profile; in irdma_sc_cqp_init()
3097 cqp->ceqs_per_vf = info->ceqs_per_vf; in irdma_sc_cqp_init()
3098 cqp->disable_packed = info->disable_packed; in irdma_sc_cqp_init()
3099 cqp->rocev2_rto_policy = info->rocev2_rto_policy; in irdma_sc_cqp_init()
3100 cqp->protocol_used = info->protocol_used; in irdma_sc_cqp_init()
3101 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); in irdma_sc_cqp_init()
3102 info->dev->cqp = cqp; in irdma_sc_cqp_init()
3104 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); in irdma_sc_cqp_init()
3105 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS] = 0; in irdma_sc_cqp_init()
3106 cqp->dev->cqp_cmd_stats[IRDMA_OP_CMPL_CMDS] = 0; in irdma_sc_cqp_init()
3108 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); in irdma_sc_cqp_init()
3110 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_sc_cqp_init()
3111 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); in irdma_sc_cqp_init()
3112 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_init()
3114 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_init()
3116 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, in irdma_sc_cqp_init()
3117 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); in irdma_sc_cqp_init()
3127 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) in irdma_sc_cqp_create() argument
3134 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; in irdma_sc_cqp_create()
3135 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size, in irdma_sc_cqp_create()
3137 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device, in irdma_sc_cqp_create()
3138 cqp->sdbuf.size, &cqp->sdbuf.pa, in irdma_sc_cqp_create()
3140 if (!cqp->sdbuf.va) in irdma_sc_cqp_create()
3143 spin_lock_init(&cqp->dev->cqp_lock); in irdma_sc_cqp_create()
3145 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | in irdma_sc_cqp_create()
3146 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | in irdma_sc_cqp_create()
3147 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | in irdma_sc_cqp_create()
3148 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); in irdma_sc_cqp_create()
3151 cqp->rocev2_rto_policy) | in irdma_sc_cqp_create()
3153 cqp->protocol_used); in irdma_sc_cqp_create()
3156 set_64bit_val(cqp->host_ctx, 0, temp); in irdma_sc_cqp_create()
3157 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa); in irdma_sc_cqp_create()
3159 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | in irdma_sc_cqp_create()
3160 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); in irdma_sc_cqp_create()
3161 set_64bit_val(cqp->host_ctx, 16, temp); in irdma_sc_cqp_create()
3162 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp); in irdma_sc_cqp_create()
3163 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | in irdma_sc_cqp_create()
3164 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); in irdma_sc_cqp_create()
3166 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | in irdma_sc_cqp_create()
3167 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); in irdma_sc_cqp_create()
3169 set_64bit_val(cqp->host_ctx, 32, temp); in irdma_sc_cqp_create()
3170 set_64bit_val(cqp->host_ctx, 40, 0); in irdma_sc_cqp_create()
3173 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | in irdma_sc_cqp_create()
3174 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | in irdma_sc_cqp_create()
3175 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); in irdma_sc_cqp_create()
3177 set_64bit_val(cqp->host_ctx, 48, temp); in irdma_sc_cqp_create()
3180 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | in irdma_sc_cqp_create()
3181 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | in irdma_sc_cqp_create()
3182 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | in irdma_sc_cqp_create()
3183 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); in irdma_sc_cqp_create()
3185 set_64bit_val(cqp->host_ctx, 56, temp); in irdma_sc_cqp_create()
3187 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false); in irdma_sc_cqp_create()
3188 p1 = cqp->host_ctx_pa >> 32; in irdma_sc_cqp_create()
3189 p2 = (u32)cqp->host_ctx_pa; in irdma_sc_cqp_create()
3191 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_create()
3192 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_create()
3195 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_create()
3199 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_create()
3200 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_create()
3203 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { in irdma_sc_cqp_create()
3208 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cqp_create()
3212 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_create()
3213 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_create()
3214 cqp->sdbuf.va = NULL; in irdma_sc_cqp_create()
3215 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_cqp_create()
3225 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_post_sq() argument
3227 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); in irdma_sc_cqp_post_sq()
3229 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_post_sq()
3231 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_cqp_post_sq()
3241 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_cqp_get_next_send_wqe_idx() argument
3247 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { in irdma_sc_cqp_get_next_send_wqe_idx()
3248 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_get_next_send_wqe_idx()
3250 cqp->sq_ring.head, cqp->sq_ring.tail, in irdma_sc_cqp_get_next_send_wqe_idx()
3251 cqp->sq_ring.size); in irdma_sc_cqp_get_next_send_wqe_idx()
3254 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); in irdma_sc_cqp_get_next_send_wqe_idx()
3258 cqp->dev->cqp_cmd_stats[IRDMA_OP_REQ_CMDS]++; in irdma_sc_cqp_get_next_send_wqe_idx()
3260 cqp->polarity = !cqp->polarity; in irdma_sc_cqp_get_next_send_wqe_idx()
3261 wqe = cqp->sq_base[*wqe_idx].elem; in irdma_sc_cqp_get_next_send_wqe_idx()
3262 cqp->scratch_array[*wqe_idx] = scratch; in irdma_sc_cqp_get_next_send_wqe_idx()
3272 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_destroy() argument
3277 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_destroy()
3278 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_destroy()
3280 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_destroy()
3284 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_destroy()
3285 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_destroy()
3286 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); in irdma_sc_cqp_destroy()
3288 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_destroy()
3289 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_destroy()
3290 cqp->sdbuf.va = NULL; in irdma_sc_cqp_destroy()
3331 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_get_cqe_info() local
3348 cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx; in irdma_sc_ccq_get_cqe_info()
3354 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_ccq_get_cqe_info()
3355 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_ccq_get_cqe_info()
3360 info->scratch = cqp->scratch_array[wqe_idx]; in irdma_sc_ccq_get_cqe_info()
3364 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1); in irdma_sc_ccq_get_cqe_info()
3366 info->cqp = cqp; in irdma_sc_ccq_get_cqe_info()
3380 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_sc_ccq_get_cqe_info()
3392 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, in irdma_sc_poll_for_cqp_op_done() argument
3400 ccq = cqp->dev->ccq; in irdma_sc_poll_for_cqp_op_done()
3402 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) in irdma_sc_poll_for_cqp_op_done()
3406 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_poll_for_cqp_op_done()
3416 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_poll_for_cqp_op_done()
3434 static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, in irdma_sc_manage_hmc_pm_func_table() argument
3441 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_hmc_pm_func_table()
3457 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_hmc_pm_func_table()
3466 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_hmc_pm_func_table()
3476 static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_commit_fpm_val_done() argument
3478 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, in irdma_sc_commit_fpm_val_done()
3491 static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_commit_fpm_val() argument
3501 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_commit_fpm_val()
3510 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_commit_fpm_val()
3518 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_commit_fpm_val()
3521 irdma_sc_cqp_post_sq(cqp); in irdma_sc_commit_fpm_val()
3523 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_commit_fpm_val()
3524 cqp->dev->hw_attrs.max_done_count); in irdma_sc_commit_fpm_val()
3526 ret_code = irdma_sc_commit_fpm_val_done(cqp); in irdma_sc_commit_fpm_val()
3537 static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_fpm_val_done() argument
3539 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, in irdma_sc_query_fpm_val_done()
3552 static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_query_fpm_val() argument
3562 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_fpm_val()
3570 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_query_fpm_val()
3577 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_query_fpm_val()
3580 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_fpm_val()
3582 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_query_fpm_val()
3583 cqp->dev->hw_attrs.max_done_count); in irdma_sc_query_fpm_val()
3585 ret_code = irdma_sc_query_fpm_val_done(cqp); in irdma_sc_query_fpm_val()
3646 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_create() local
3650 cqp = ceq->dev->cqp; in irdma_sc_ceq_create()
3651 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_create()
3668 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_create()
3676 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_create()
3687 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_create_done() local
3689 cqp = ceq->dev->cqp; in irdma_sc_cceq_create_done()
3690 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, in irdma_sc_cceq_create_done()
3700 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_destroy_done() local
3705 cqp = ceq->dev->cqp; in irdma_sc_cceq_destroy_done()
3706 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cceq_destroy_done()
3708 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, in irdma_sc_cceq_destroy_done()
3744 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_destroy() local
3748 cqp = ceq->dev->cqp; in irdma_sc_ceq_destroy()
3749 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_destroy()
3760 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_destroy()
3768 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_destroy()
3911 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_create() local
3914 cqp = aeq->dev->cqp; in irdma_sc_aeq_create()
3915 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_create()
3927 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_create()
3935 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_create()
3950 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_destroy() local
3957 cqp = dev->cqp; in irdma_sc_aeq_destroy()
3958 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_destroy()
3966 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_destroy()
3974 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_destroy()
4196 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_create_done() local
4198 cqp = ccq->dev->cqp; in irdma_sc_ccq_create_done()
4200 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); in irdma_sc_ccq_create_done()
4224 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; in irdma_sc_ccq_create()
4237 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_destroy() local
4243 cqp = ccq->dev->cqp; in irdma_sc_ccq_destroy()
4244 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ccq_destroy()
4260 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ccq_destroy()
4267 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_ccq_destroy()
4270 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ccq_destroy()
4271 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_ccq_destroy()
4272 cqp->dev->hw_attrs.max_done_count); in irdma_sc_ccq_destroy()
4275 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_ccq_destroy()
4300 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_init_iw_hmc()
4371 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_cfg_iw_fpm()
4390 static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, in cqp_sds_wqe_fill() argument
4397 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; in cqp_sds_wqe_fill()
4401 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); in cqp_sds_wqe_fill()
4445 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in cqp_sds_wqe_fill()
4472 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_pe_sds() local
4475 ret_code = cqp_sds_wqe_fill(cqp, info, scratch); in irdma_update_pe_sds()
4477 irdma_sc_cqp_post_sq(cqp); in irdma_update_pe_sds()
4491 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_sds_noccq() local
4494 ret_code = cqp_sds_wqe_fill(cqp, info, 0); in irdma_update_sds_noccq()
4498 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_update_sds_noccq()
4500 irdma_sc_cqp_post_sq(cqp); in irdma_update_sds_noccq()
4501 return irdma_cqp_poll_registers(cqp, tail, in irdma_update_sds_noccq()
4502 cqp->dev->hw_attrs.max_done_count); in irdma_update_sds_noccq()
4513 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_static_hmc_pages_allocated() argument
4521 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_static_hmc_pages_allocated()
4530 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_static_hmc_pages_allocated()
4538 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_static_hmc_pages_allocated()
4541 irdma_sc_cqp_post_sq(cqp); in irdma_sc_static_hmc_pages_allocated()
4544 return irdma_cqp_poll_registers(cqp, tail, in irdma_sc_static_hmc_pages_allocated()
4545 cqp->dev->hw_attrs.max_done_count); in irdma_sc_static_hmc_pages_allocated()
4547 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_static_hmc_pages_allocated()
4559 static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) in irdma_cqp_ring_full() argument
4561 return IRDMA_RING_FULL_ERR(cqp->sq_ring); in irdma_cqp_ring_full()
4598 static int irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_rdma_features_done() argument
4600 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_query_rdma_features_done()
4611 static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, in irdma_sc_query_rdma_features() argument
4617 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_rdma_features()
4625 cqp->polarity) | in irdma_sc_query_rdma_features()
4634 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_rdma_features()
4657 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4659 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4682 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4684 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4726 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) in irdma_q1_cnt()
4881 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && in irdma_cfg_fpm_val()
5022 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, in irdma_exec_cqp_cmd()
5035 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, in irdma_exec_cqp_cmd()
5041 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5046 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5053 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, in irdma_exec_cqp_cmd()
5061 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, in irdma_exec_cqp_cmd()
5072 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, in irdma_exec_cqp_cmd()
5078 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, in irdma_exec_cqp_cmd()
5083 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5089 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5095 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5101 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, in irdma_exec_cqp_cmd()
5106 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, in irdma_exec_cqp_cmd()
5111 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5117 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, in irdma_exec_cqp_cmd()
5123 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, in irdma_exec_cqp_cmd()
5172 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5178 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5183 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5189 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5196 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, in irdma_exec_cqp_cmd()
5201 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, in irdma_exec_cqp_cmd()
5206 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, in irdma_exec_cqp_cmd()
5211 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, in irdma_exec_cqp_cmd()
5216 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, in irdma_exec_cqp_cmd()
5240 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) in irdma_process_cqp_cmd()
5260 !irdma_cqp_ring_full(dev->cqp)) { in irdma_process_bh()