Lines Matching refs:cqp
182 static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_arp_cache_entry() argument
189 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_arp_cache_entry()
199 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_arp_cache_entry()
207 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_arp_cache_entry()
219 static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_arp_cache_entry() argument
225 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_arp_cache_entry()
231 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_del_arp_cache_entry()
240 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_arp_cache_entry()
252 static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_apbvt_entry() argument
259 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_apbvt_entry()
267 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_apbvt_entry()
275 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_apbvt_entry()
300 irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp, in irdma_sc_manage_qhash_table_entry() argument
310 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_qhash_table_entry()
351 temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) | in irdma_sc_manage_qhash_table_entry()
365 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_qhash_table_entry()
451 struct irdma_sc_cqp *cqp; in irdma_sc_qp_create() local
455 cqp = qp->dev->cqp; in irdma_sc_qp_create()
456 if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id || in irdma_sc_qp_create()
457 qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt) in irdma_sc_qp_create()
460 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_create()
479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_create()
487 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_create()
503 struct irdma_sc_cqp *cqp; in irdma_sc_qp_modify() local
508 cqp = qp->dev->cqp; in irdma_sc_qp_modify()
509 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_modify()
548 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_modify()
556 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_modify()
573 struct irdma_sc_cqp *cqp; in irdma_sc_qp_destroy() local
576 cqp = qp->dev->cqp; in irdma_sc_qp_destroy()
577 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_destroy()
589 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_destroy()
597 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_destroy()
761 static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_alloc_local_mac_entry() argument
767 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_local_mac_entry()
773 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_local_mac_entry()
784 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_local_mac_entry()
795 static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp, in irdma_sc_add_local_mac_entry() argument
802 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_add_local_mac_entry()
811 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_add_local_mac_entry()
821 irdma_sc_cqp_post_sq(cqp); in irdma_sc_add_local_mac_entry()
833 static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_del_local_mac_entry() argument
840 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_del_local_mac_entry()
847 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_del_local_mac_entry()
859 irdma_sc_cqp_post_sq(cqp); in irdma_sc_del_local_mac_entry()
1060 struct irdma_sc_cqp *cqp; in irdma_sc_alloc_stag() local
1074 cqp = dev->cqp; in irdma_sc_alloc_stag()
1075 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_alloc_stag()
1099 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_alloc_stag()
1107 irdma_sc_cqp_post_sq(cqp); in irdma_sc_alloc_stag()
1125 struct irdma_sc_cqp *cqp; in irdma_sc_mr_reg_non_shared() local
1154 cqp = dev->cqp; in irdma_sc_mr_reg_non_shared()
1155 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mr_reg_non_shared()
1190 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mr_reg_non_shared()
1198 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mr_reg_non_shared()
1216 struct irdma_sc_cqp *cqp; in irdma_sc_dealloc_stag() local
1218 cqp = dev->cqp; in irdma_sc_dealloc_stag()
1219 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_dealloc_stag()
1230 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_dealloc_stag()
1238 irdma_sc_cqp_post_sq(cqp); in irdma_sc_dealloc_stag()
1255 struct irdma_sc_cqp *cqp; in irdma_sc_mw_alloc() local
1258 cqp = dev->cqp; in irdma_sc_mw_alloc()
1259 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_mw_alloc()
1272 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_mw_alloc()
1280 irdma_sc_cqp_post_sq(cqp); in irdma_sc_mw_alloc()
2047 static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp, in irdma_sc_gather_stats() argument
2057 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gather_stats()
2065 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_gather_stats()
2079 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gather_stats()
2080 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_gather_stats()
2082 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_gather_stats()
2094 static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp, in irdma_sc_manage_stats_inst() argument
2101 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_stats_inst()
2107 temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) | in irdma_sc_manage_stats_inst()
2121 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_stats_inst()
2131 static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp, in irdma_sc_set_up_map() argument
2138 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_set_up_map()
2150 temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) | in irdma_sc_set_up_map()
2161 irdma_sc_cqp_post_sq(cqp); in irdma_sc_set_up_map()
2173 static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp, in irdma_sc_manage_ws_node() argument
2180 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_ws_node()
2188 temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) | in irdma_sc_manage_ws_node()
2203 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_ws_node()
2221 struct irdma_sc_cqp *cqp; in irdma_sc_qp_flush_wqes() local
2239 cqp = qp->pd->dev->cqp; in irdma_sc_qp_flush_wqes()
2240 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_flush_wqes()
2269 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_flush_wqes()
2277 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_flush_wqes()
2295 struct irdma_sc_cqp *cqp; in irdma_sc_gen_ae() local
2298 cqp = qp->pd->dev->cqp; in irdma_sc_gen_ae()
2299 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_gen_ae()
2310 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_gen_ae()
2318 irdma_sc_cqp_post_sq(cqp); in irdma_sc_gen_ae()
2334 struct irdma_sc_cqp *cqp; in irdma_sc_qp_upload_context() local
2337 cqp = dev->cqp; in irdma_sc_qp_upload_context()
2338 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_qp_upload_context()
2349 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_qp_upload_context()
2357 irdma_sc_cqp_post_sq(cqp); in irdma_sc_qp_upload_context()
2369 static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp, in irdma_sc_manage_push_page() argument
2377 info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages) in irdma_sc_manage_push_page()
2380 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_push_page()
2388 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in irdma_sc_manage_push_page()
2397 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_push_page()
2408 static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, in irdma_sc_suspend_qp() argument
2414 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_suspend_qp()
2420 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_suspend_qp()
2427 irdma_sc_cqp_post_sq(cqp); in irdma_sc_suspend_qp()
2438 static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp, in irdma_sc_resume_qp() argument
2444 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_resume_qp()
2453 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_resume_qp()
2460 irdma_sc_cqp_post_sq(cqp); in irdma_sc_resume_qp()
2520 struct irdma_sc_cqp *cqp; in irdma_sc_cq_create() local
2525 cqp = cq->dev->cqp; in irdma_sc_cq_create()
2526 if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt) in irdma_sc_cq_create()
2539 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_create()
2570 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_create()
2578 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_create()
2591 struct irdma_sc_cqp *cqp; in irdma_sc_cq_destroy() local
2596 cqp = cq->dev->cqp; in irdma_sc_cq_destroy()
2597 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_destroy()
2621 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_destroy()
2629 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_destroy()
2659 struct irdma_sc_cqp *cqp; in irdma_sc_cq_modify() local
2669 cqp = cq->dev->cqp; in irdma_sc_cq_modify()
2670 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_cq_modify()
2695 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_cq_modify()
2703 irdma_sc_cqp_post_sq(cqp); in irdma_sc_cq_modify()
2715 u64 completed_ops = atomic64_read(&dev->cqp->completed_ops); in irdma_check_cqp_progress()
2720 } else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) { in irdma_check_cqp_progress()
2732 static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val, in irdma_get_cqp_reg_info() argument
2735 *val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_get_cqp_reg_info()
2746 static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail, in irdma_cqp_poll_registers() argument
2753 irdma_get_cqp_reg_info(cqp, &val, &newtail, &error); in irdma_cqp_poll_registers()
2755 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_cqp_poll_registers()
2756 ibdev_dbg(to_ibdev(cqp->dev), in irdma_cqp_poll_registers()
2763 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_cqp_poll_registers()
2764 atomic64_inc(&cqp->completed_ops); in irdma_cqp_poll_registers()
2767 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_cqp_poll_registers()
3088 int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp, in irdma_sc_cqp_init() argument
3100 cqp->size = sizeof(*cqp); in irdma_sc_cqp_init()
3101 cqp->sq_size = info->sq_size; in irdma_sc_cqp_init()
3102 cqp->hw_sq_size = hw_sq_size; in irdma_sc_cqp_init()
3103 cqp->sq_base = info->sq; in irdma_sc_cqp_init()
3104 cqp->host_ctx = info->host_ctx; in irdma_sc_cqp_init()
3105 cqp->sq_pa = info->sq_pa; in irdma_sc_cqp_init()
3106 cqp->host_ctx_pa = info->host_ctx_pa; in irdma_sc_cqp_init()
3107 cqp->dev = info->dev; in irdma_sc_cqp_init()
3108 cqp->struct_ver = info->struct_ver; in irdma_sc_cqp_init()
3109 cqp->hw_maj_ver = info->hw_maj_ver; in irdma_sc_cqp_init()
3110 cqp->hw_min_ver = info->hw_min_ver; in irdma_sc_cqp_init()
3111 cqp->scratch_array = info->scratch_array; in irdma_sc_cqp_init()
3112 cqp->polarity = 0; in irdma_sc_cqp_init()
3113 cqp->en_datacenter_tcp = info->en_datacenter_tcp; in irdma_sc_cqp_init()
3114 cqp->ena_vf_count = info->ena_vf_count; in irdma_sc_cqp_init()
3115 cqp->hmc_profile = info->hmc_profile; in irdma_sc_cqp_init()
3116 cqp->ceqs_per_vf = info->ceqs_per_vf; in irdma_sc_cqp_init()
3117 cqp->disable_packed = info->disable_packed; in irdma_sc_cqp_init()
3118 cqp->rocev2_rto_policy = info->rocev2_rto_policy; in irdma_sc_cqp_init()
3119 cqp->protocol_used = info->protocol_used; in irdma_sc_cqp_init()
3120 memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params)); in irdma_sc_cqp_init()
3121 info->dev->cqp = cqp; in irdma_sc_cqp_init()
3123 IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size); in irdma_sc_cqp_init()
3124 cqp->requested_ops = 0; in irdma_sc_cqp_init()
3125 atomic64_set(&cqp->completed_ops, 0); in irdma_sc_cqp_init()
3127 INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head); in irdma_sc_cqp_init()
3129 writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]); in irdma_sc_cqp_init()
3130 writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]); in irdma_sc_cqp_init()
3131 writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_init()
3133 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_init()
3135 cqp->sq_size, cqp->hw_sq_size, cqp->sq_base, in irdma_sc_cqp_init()
3136 (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity); in irdma_sc_cqp_init()
3146 int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err) in irdma_sc_cqp_create() argument
3153 hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev; in irdma_sc_cqp_create()
3154 cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size, in irdma_sc_cqp_create()
3156 cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device, in irdma_sc_cqp_create()
3157 cqp->sdbuf.size, &cqp->sdbuf.pa, in irdma_sc_cqp_create()
3159 if (!cqp->sdbuf.va) in irdma_sc_cqp_create()
3162 spin_lock_init(&cqp->dev->cqp_lock); in irdma_sc_cqp_create()
3164 temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) | in irdma_sc_cqp_create()
3165 FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) | in irdma_sc_cqp_create()
3166 FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) | in irdma_sc_cqp_create()
3167 FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf); in irdma_sc_cqp_create()
3170 cqp->rocev2_rto_policy) | in irdma_sc_cqp_create()
3172 cqp->protocol_used); in irdma_sc_cqp_create()
3175 set_64bit_val(cqp->host_ctx, 0, temp); in irdma_sc_cqp_create()
3176 set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa); in irdma_sc_cqp_create()
3178 temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) | in irdma_sc_cqp_create()
3179 FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile); in irdma_sc_cqp_create()
3180 set_64bit_val(cqp->host_ctx, 16, temp); in irdma_sc_cqp_create()
3181 set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp); in irdma_sc_cqp_create()
3182 temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) | in irdma_sc_cqp_create()
3183 FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver); in irdma_sc_cqp_create()
3185 temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) | in irdma_sc_cqp_create()
3186 FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor); in irdma_sc_cqp_create()
3188 set_64bit_val(cqp->host_ctx, 32, temp); in irdma_sc_cqp_create()
3189 set_64bit_val(cqp->host_ctx, 40, 0); in irdma_sc_cqp_create()
3192 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) | in irdma_sc_cqp_create()
3193 FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) | in irdma_sc_cqp_create()
3194 FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor); in irdma_sc_cqp_create()
3196 set_64bit_val(cqp->host_ctx, 48, temp); in irdma_sc_cqp_create()
3199 temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) | in irdma_sc_cqp_create()
3200 FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) | in irdma_sc_cqp_create()
3201 FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) | in irdma_sc_cqp_create()
3202 FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod); in irdma_sc_cqp_create()
3204 set_64bit_val(cqp->host_ctx, 56, temp); in irdma_sc_cqp_create()
3206 8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false); in irdma_sc_cqp_create()
3207 p1 = cqp->host_ctx_pa >> 32; in irdma_sc_cqp_create()
3208 p2 = (u32)cqp->host_ctx_pa; in irdma_sc_cqp_create()
3210 writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_create()
3211 writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_create()
3214 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_create()
3218 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_create()
3219 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_create()
3222 if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) { in irdma_sc_cqp_create()
3227 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cqp_create()
3231 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_create()
3232 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_create()
3233 cqp->sdbuf.va = NULL; in irdma_sc_cqp_create()
3234 err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_cqp_create()
3244 void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_post_sq() argument
3246 writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db); in irdma_sc_cqp_post_sq()
3248 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_post_sq()
3250 cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size); in irdma_sc_cqp_post_sq()
3260 __le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_cqp_get_next_send_wqe_idx() argument
3266 if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) { in irdma_sc_cqp_get_next_send_wqe_idx()
3267 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_cqp_get_next_send_wqe_idx()
3269 cqp->sq_ring.head, cqp->sq_ring.tail, in irdma_sc_cqp_get_next_send_wqe_idx()
3270 cqp->sq_ring.size); in irdma_sc_cqp_get_next_send_wqe_idx()
3273 IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code); in irdma_sc_cqp_get_next_send_wqe_idx()
3277 cqp->requested_ops++; in irdma_sc_cqp_get_next_send_wqe_idx()
3279 cqp->polarity = !cqp->polarity; in irdma_sc_cqp_get_next_send_wqe_idx()
3280 wqe = cqp->sq_base[*wqe_idx].elem; in irdma_sc_cqp_get_next_send_wqe_idx()
3281 cqp->scratch_array[*wqe_idx] = scratch; in irdma_sc_cqp_get_next_send_wqe_idx()
3291 int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp) in irdma_sc_cqp_destroy() argument
3296 writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]); in irdma_sc_cqp_destroy()
3297 writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]); in irdma_sc_cqp_destroy()
3299 if (cnt++ > cqp->dev->hw_attrs.max_done_count) { in irdma_sc_cqp_destroy()
3303 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_cqp_destroy()
3304 val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]); in irdma_sc_cqp_destroy()
3305 } while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE)); in irdma_sc_cqp_destroy()
3307 dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size, in irdma_sc_cqp_destroy()
3308 cqp->sdbuf.va, cqp->sdbuf.pa); in irdma_sc_cqp_destroy()
3309 cqp->sdbuf.va = NULL; in irdma_sc_cqp_destroy()
3350 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_get_cqe_info() local
3370 cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx; in irdma_sc_ccq_get_cqe_info()
3376 error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]); in irdma_sc_ccq_get_cqe_info()
3377 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_ccq_get_cqe_info()
3382 info->scratch = cqp->scratch_array[wqe_idx]; in irdma_sc_ccq_get_cqe_info()
3386 get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1); in irdma_sc_ccq_get_cqe_info()
3388 info->cqp = cqp; in irdma_sc_ccq_get_cqe_info()
3402 IRDMA_RING_MOVE_TAIL(cqp->sq_ring); in irdma_sc_ccq_get_cqe_info()
3403 atomic64_inc(&cqp->completed_ops); in irdma_sc_ccq_get_cqe_info()
3414 int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code, in irdma_sc_poll_for_cqp_op_done() argument
3422 ccq = cqp->dev->ccq; in irdma_sc_poll_for_cqp_op_done()
3424 if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count) in irdma_sc_poll_for_cqp_op_done()
3428 udelay(cqp->dev->hw_attrs.max_sleep_count); in irdma_sc_poll_for_cqp_op_done()
3438 ibdev_dbg(to_ibdev(cqp->dev), in irdma_sc_poll_for_cqp_op_done()
3456 static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp, in irdma_sc_manage_hmc_pm_func_table() argument
3463 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_manage_hmc_pm_func_table()
3479 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_manage_hmc_pm_func_table()
3488 irdma_sc_cqp_post_sq(cqp); in irdma_sc_manage_hmc_pm_func_table()
3498 static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_commit_fpm_val_done() argument
3500 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL, in irdma_sc_commit_fpm_val_done()
3513 static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_commit_fpm_val() argument
3523 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_commit_fpm_val()
3532 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_commit_fpm_val()
3540 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_commit_fpm_val()
3543 irdma_sc_cqp_post_sq(cqp); in irdma_sc_commit_fpm_val()
3545 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_commit_fpm_val()
3546 cqp->dev->hw_attrs.max_done_count); in irdma_sc_commit_fpm_val()
3548 ret_code = irdma_sc_commit_fpm_val_done(cqp); in irdma_sc_commit_fpm_val()
3559 static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_fpm_val_done() argument
3561 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL, in irdma_sc_query_fpm_val_done()
3574 static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_query_fpm_val() argument
3584 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_fpm_val()
3592 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_query_fpm_val()
3599 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_query_fpm_val()
3602 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_fpm_val()
3604 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_query_fpm_val()
3605 cqp->dev->hw_attrs.max_done_count); in irdma_sc_query_fpm_val()
3607 ret_code = irdma_sc_query_fpm_val_done(cqp); in irdma_sc_query_fpm_val()
3668 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_create() local
3672 cqp = ceq->dev->cqp; in irdma_sc_ceq_create()
3673 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_create()
3690 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_create()
3698 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_create()
3709 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_create_done() local
3711 cqp = ceq->dev->cqp; in irdma_sc_cceq_create_done()
3712 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ, in irdma_sc_cceq_create_done()
3722 struct irdma_sc_cqp *cqp; in irdma_sc_cceq_destroy_done() local
3727 cqp = ceq->dev->cqp; in irdma_sc_cceq_destroy_done()
3728 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_cceq_destroy_done()
3730 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ, in irdma_sc_cceq_destroy_done()
3766 struct irdma_sc_cqp *cqp; in irdma_sc_ceq_destroy() local
3770 cqp = ceq->dev->cqp; in irdma_sc_ceq_destroy()
3771 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ceq_destroy()
3782 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ceq_destroy()
3790 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ceq_destroy()
3933 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_create() local
3936 cqp = aeq->dev->cqp; in irdma_sc_aeq_create()
3937 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_create()
3949 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_create()
3957 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_create()
3972 struct irdma_sc_cqp *cqp; in irdma_sc_aeq_destroy() local
3979 cqp = dev->cqp; in irdma_sc_aeq_destroy()
3980 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_aeq_destroy()
3988 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_aeq_destroy()
3996 irdma_sc_cqp_post_sq(cqp); in irdma_sc_aeq_destroy()
4219 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_create_done() local
4221 cqp = ccq->dev->cqp; in irdma_sc_ccq_create_done()
4223 return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL); in irdma_sc_ccq_create_done()
4247 ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd; in irdma_sc_ccq_create()
4260 struct irdma_sc_cqp *cqp; in irdma_sc_ccq_destroy() local
4266 cqp = ccq->dev->cqp; in irdma_sc_ccq_destroy()
4267 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_ccq_destroy()
4283 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_ccq_destroy()
4290 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_ccq_destroy()
4293 irdma_sc_cqp_post_sq(cqp); in irdma_sc_ccq_destroy()
4294 ret_code = irdma_cqp_poll_registers(cqp, tail, in irdma_sc_ccq_destroy()
4295 cqp->dev->hw_attrs.max_done_count); in irdma_sc_ccq_destroy()
4298 cqp->process_cqp_sds = irdma_update_sds_noccq; in irdma_sc_ccq_destroy()
4323 ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_init_iw_hmc()
4394 ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id, in irdma_sc_cfg_iw_fpm()
4413 static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp, in cqp_sds_wqe_fill() argument
4420 struct irdma_dma_mem *sdbuf = &cqp->sdbuf; in cqp_sds_wqe_fill()
4424 wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx); in cqp_sds_wqe_fill()
4468 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) | in cqp_sds_wqe_fill()
4495 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_pe_sds() local
4498 ret_code = cqp_sds_wqe_fill(cqp, info, scratch); in irdma_update_pe_sds()
4500 irdma_sc_cqp_post_sq(cqp); in irdma_update_pe_sds()
4514 struct irdma_sc_cqp *cqp = dev->cqp; in irdma_update_sds_noccq() local
4517 ret_code = cqp_sds_wqe_fill(cqp, info, 0); in irdma_update_sds_noccq()
4521 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_update_sds_noccq()
4523 irdma_sc_cqp_post_sq(cqp); in irdma_update_sds_noccq()
4524 return irdma_cqp_poll_registers(cqp, tail, in irdma_update_sds_noccq()
4525 cqp->dev->hw_attrs.max_done_count); in irdma_update_sds_noccq()
4536 int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch, in irdma_sc_static_hmc_pages_allocated() argument
4544 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_static_hmc_pages_allocated()
4553 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity); in irdma_sc_static_hmc_pages_allocated()
4561 irdma_get_cqp_reg_info(cqp, &val, &tail, &error); in irdma_sc_static_hmc_pages_allocated()
4564 irdma_sc_cqp_post_sq(cqp); in irdma_sc_static_hmc_pages_allocated()
4567 return irdma_cqp_poll_registers(cqp, tail, in irdma_sc_static_hmc_pages_allocated()
4568 cqp->dev->hw_attrs.max_done_count); in irdma_sc_static_hmc_pages_allocated()
4570 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_static_hmc_pages_allocated()
4582 static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp) in irdma_cqp_ring_full() argument
4584 return IRDMA_RING_FULL_ERR(cqp->sq_ring); in irdma_cqp_ring_full()
4621 static int irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp) in irdma_sc_query_rdma_features_done() argument
4623 return irdma_sc_poll_for_cqp_op_done(cqp, in irdma_sc_query_rdma_features_done()
4634 static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp, in irdma_sc_query_rdma_features() argument
4640 wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch); in irdma_sc_query_rdma_features()
4648 cqp->polarity) | in irdma_sc_query_rdma_features()
4657 irdma_sc_cqp_post_sq(cqp); in irdma_sc_query_rdma_features()
4680 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4682 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4705 ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0); in irdma_get_rdma_features()
4707 ret_code = irdma_sc_query_rdma_features_done(dev->cqp); in irdma_get_rdma_features()
4749 if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) in irdma_q1_cnt()
4906 if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF && in irdma_cfg_fpm_val()
5047 status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp, in irdma_exec_cqp_cmd()
5060 irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp, in irdma_exec_cqp_cmd()
5066 status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5071 status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp, in irdma_exec_cqp_cmd()
5078 status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp, in irdma_exec_cqp_cmd()
5086 status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp, in irdma_exec_cqp_cmd()
5097 status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp, in irdma_exec_cqp_cmd()
5103 status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp, in irdma_exec_cqp_cmd()
5108 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5114 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5120 status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp, in irdma_exec_cqp_cmd()
5126 status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp, in irdma_exec_cqp_cmd()
5131 status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp, in irdma_exec_cqp_cmd()
5136 status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5142 status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp, in irdma_exec_cqp_cmd()
5148 status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp, in irdma_exec_cqp_cmd()
5197 status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp, in irdma_exec_cqp_cmd()
5203 status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5208 status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5214 status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp, in irdma_exec_cqp_cmd()
5221 status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp, in irdma_exec_cqp_cmd()
5226 status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp, in irdma_exec_cqp_cmd()
5231 status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp, in irdma_exec_cqp_cmd()
5236 status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp, in irdma_exec_cqp_cmd()
5241 status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp, in irdma_exec_cqp_cmd()
5265 if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp)) in irdma_process_cqp_cmd()
5285 !irdma_cqp_ring_full(dev->cqp)) { in irdma_process_bh()