40 	if (cmpxchg(&ibmr->u.frmr.fr_state,  in rds_transition_frwr_state()
58 	struct rds_ib_frmr *frmr;  in rds_ib_alloc_frmr()  local
77 	frmr = &ibmr->u.frmr;  in rds_ib_alloc_frmr()
78 	frmr->mr = ib_alloc_mr(rds_ibdev->pd, IB_MR_TYPE_MEM_REG,  in rds_ib_alloc_frmr()
80 	if (IS_ERR(frmr->mr)) {  in rds_ib_alloc_frmr()
82 		err = PTR_ERR(frmr->mr);  in rds_ib_alloc_frmr()
95 	frmr->fr_state = FRMR_IS_FREE;  in rds_ib_alloc_frmr()
96 	init_waitqueue_head(&frmr->fr_inv_done);  in rds_ib_alloc_frmr()
97 	init_waitqueue_head(&frmr->fr_reg_done);  in rds_ib_alloc_frmr()
125 	struct rds_ib_frmr *frmr = &ibmr->u.frmr;  in rds_ib_post_reg_frmr()  local
134 	ret = ib_map_mr_sg_zbva(frmr->mr, ibmr->sg, ibmr->sg_dma_len,  in rds_ib_post_reg_frmr()
139 	if (cmpxchg(&frmr->fr_state,  in rds_ib_post_reg_frmr()
150 	ib_update_fast_reg_key(frmr->mr, ibmr->remap_count++);  in rds_ib_post_reg_frmr()
151 	frmr->fr_reg = true;  in rds_ib_post_reg_frmr()
157 	reg_wr.mr = frmr->mr;  in rds_ib_post_reg_frmr()
158 	reg_wr.key = frmr->mr->rkey;  in rds_ib_post_reg_frmr()
180 	wait_event(frmr->fr_reg_done, !frmr->fr_reg);  in rds_ib_post_reg_frmr()
193 	struct rds_ib_frmr *frmr = &ibmr->u.frmr;  in rds_ib_map_frmr()  local
206 	frmr->sg_byte_len = 0;  in rds_ib_map_frmr()
215 	frmr->sg_byte_len = 0;  in rds_ib_map_frmr()
216 	frmr->dma_npages = 0;  in rds_ib_map_frmr()
224 		frmr->sg_byte_len += dma_len;  in rds_ib_map_frmr()
229 				++frmr->dma_npages;  in rds_ib_map_frmr()
236 				++frmr->dma_npages;  in rds_ib_map_frmr()
241 	frmr->dma_npages += len >> PAGE_SHIFT;  in rds_ib_map_frmr()
243 	if (frmr->dma_npages > ibmr->pool->max_pages) {  in rds_ib_map_frmr()
269 	struct rds_ib_frmr *frmr = &ibmr->u.frmr;  in rds_ib_post_inv()  local
273 	if (!i_cm_id || !i_cm_id->qp || !frmr->mr)  in rds_ib_post_inv()
276 	if (frmr->fr_state != FRMR_IS_INUSE)  in rds_ib_post_inv()
284 	frmr->fr_inv = true;  in rds_ib_post_inv()
285 	s_wr = &frmr->fr_wr;  in rds_ib_post_inv()
290 	s_wr->ex.invalidate_rkey = frmr->mr->rkey;  in rds_ib_post_inv()
296 		frmr->fr_inv = false;  in rds_ib_post_inv()
316 	wait_event(frmr->fr_inv_done, frmr->fr_state != FRMR_IS_INUSE);  in rds_ib_post_inv()
325 	struct rds_ib_frmr *frmr = &ibmr->u.frmr;  in rds_ib_mr_cqe_handler()  local
339 	if (frmr->fr_inv) {  in rds_ib_mr_cqe_handler()
341 		frmr->fr_inv = false;  in rds_ib_mr_cqe_handler()
342 		wake_up(&frmr->fr_inv_done);  in rds_ib_mr_cqe_handler()
345 	if (frmr->fr_reg) {  in rds_ib_mr_cqe_handler()
346 		frmr->fr_reg = false;  in rds_ib_mr_cqe_handler()
347 		wake_up(&frmr->fr_reg_done);  in rds_ib_mr_cqe_handler()
361 	struct rds_ib_frmr *frmr;  in rds_ib_unreg_frmr()  local
380 		frmr = &ibmr->u.frmr;  in rds_ib_unreg_frmr()
382 		if (freed < goal || frmr->fr_state == FRMR_IS_STALE) {  in rds_ib_unreg_frmr()
384 			if (frmr->fr_state == FRMR_IS_INUSE)  in rds_ib_unreg_frmr()
392 			if (frmr->mr)  in rds_ib_unreg_frmr()
393 				ib_dereg_mr(frmr->mr);  in rds_ib_unreg_frmr()
407 	struct rds_ib_frmr *frmr;  in rds_ib_reg_frmr()  local
421 		frmr = &ibmr->u.frmr;  in rds_ib_reg_frmr()
422 	} while (frmr->fr_state != FRMR_IS_FREE);  in rds_ib_reg_frmr()
428 		*key = frmr->mr->rkey;  in rds_ib_reg_frmr()
440 	struct rds_ib_frmr *frmr = &ibmr->u.frmr;  in rds_ib_free_frmr_list()  local
442 	if (frmr->fr_state == FRMR_IS_STALE)  in rds_ib_free_frmr_list()