13 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,
52 			   struct hfi1_ctxtdata *uctxt)  in hfi1_user_exp_rcv_init()  argument
56 	fd->entry_to_rb = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
62 	if (!HFI1_CAP_UGET_MASK(uctxt->flags, TID_UNMAP)) {  in hfi1_user_exp_rcv_init()
64 		fd->invalid_tids = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
89 	if (uctxt->subctxt_cnt && fd->use_mn) {  in hfi1_user_exp_rcv_init()
92 		fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
93 		remainder = uctxt->expected_count % uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
97 		fd->tid_limit = uctxt->expected_count;  in hfi1_user_exp_rcv_init()
106 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_free()  local
108 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
109 	if (!EXP_TID_SET_EMPTY(uctxt->tid_full_list))  in hfi1_user_exp_rcv_free()
110 		unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);  in hfi1_user_exp_rcv_free()
111 	if (!EXP_TID_SET_EMPTY(uctxt->tid_used_list))  in hfi1_user_exp_rcv_free()
112 		unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);  in hfi1_user_exp_rcv_free()
113 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_free()
141 	struct hfi1_devdata *dd = fd->uctxt->dd;  in unpin_rcv_pages()
166 	struct hfi1_devdata *dd = fd->uctxt->dd;  in pin_rcv_pages()
173 	if (npages > fd->uctxt->expected_count) {  in pin_rcv_pages()
257 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_setup()  local
258 	struct hfi1_devdata *dd = uctxt->dd;  in hfi1_user_exp_rcv_setup()
277 	tidbuf->psets = kcalloc(uctxt->expected_count, sizeof(*tidbuf->psets),  in hfi1_user_exp_rcv_setup()
330 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
335 	while (ngroups && uctxt->tid_group_list.count) {  in hfi1_user_exp_rcv_setup()
337 			tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
348 			tid_group_add_tail(grp, &uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
354 		tid_group_add_tail(grp, &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
367 		if (!uctxt->tid_used_list.count || need_group) {  in hfi1_user_exp_rcv_setup()
368 			if (!uctxt->tid_group_list.count)  in hfi1_user_exp_rcv_setup()
371 			grp = tid_group_pop(&uctxt->tid_group_list);  in hfi1_user_exp_rcv_setup()
372 			tid_group_add_tail(grp, &uctxt->tid_used_list);  in hfi1_user_exp_rcv_setup()
380 		list_for_each_entry_safe(grp, ptr, &uctxt->tid_used_list.list,  in hfi1_user_exp_rcv_setup()
396 						       &uctxt->tid_used_list,  in hfi1_user_exp_rcv_setup()
397 						       &uctxt->tid_full_list);  in hfi1_user_exp_rcv_setup()
416 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_setup()
495 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_clear()  local
507 	mutex_lock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
520 	mutex_unlock(&uctxt->exp_mutex);  in hfi1_user_exp_rcv_clear()
529 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_invalid()  local
530 	unsigned long *ev = uctxt->dd->events +  in hfi1_user_exp_rcv_invalid()
531 		(uctxt_offset(uctxt) + fd->subctxt);  in hfi1_user_exp_rcv_invalid()
541 	array = kcalloc(uctxt->expected_count, sizeof(*array), GFP_KERNEL);  in hfi1_user_exp_rcv_invalid()
671 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in program_rcvarray()  local
672 	struct hfi1_devdata *dd = uctxt->dd;  in program_rcvarray()
718 		tidinfo = rcventry2tidinfo(rcventry - uctxt->expected_base) |  in program_rcvarray()
739 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in set_rcvarray_entry()  local
741 	struct hfi1_devdata *dd = uctxt->dd;  in set_rcvarray_entry()
780 	fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;  in set_rcvarray_entry()
783 	trace_hfi1_exp_tid_reg(uctxt->ctxt, fd->subctxt, rcventry, npages,  in set_rcvarray_entry()
800 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in unprogram_rcvarray()  local
801 	struct hfi1_devdata *dd = uctxt->dd;  in unprogram_rcvarray()
806 	if (tididx >= uctxt->expected_count) {  in unprogram_rcvarray()
808 			   tididx, uctxt->ctxt);  in unprogram_rcvarray()
818 	if (!node || node->rcventry != (uctxt->expected_base + rcventry))  in unprogram_rcvarray()
830 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in __clear_tid_node()  local
831 	struct hfi1_devdata *dd = uctxt->dd;  in __clear_tid_node()
838 	trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,  in __clear_tid_node()
853 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in clear_tid_node()  local
861 		tid_group_move(node->grp, &uctxt->tid_full_list,  in clear_tid_node()
862 			       &uctxt->tid_used_list);  in clear_tid_node()
864 		tid_group_move(node->grp, &uctxt->tid_used_list,  in clear_tid_node()
865 			       &uctxt->tid_group_list);  in clear_tid_node()
873 static void unlock_exp_tids(struct hfi1_ctxtdata *uctxt,  in unlock_exp_tids()  argument
889 							  uctxt->expected_base];  in unlock_exp_tids()
909 	struct hfi1_ctxtdata *uctxt = fdata->uctxt;  in tid_rb_invalidate()  local
918 	trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,  in tid_rb_invalidate()
926 	if (fdata->invalid_tid_idx < uctxt->expected_count) {  in tid_rb_invalidate()
928 			rcventry2tidinfo(node->rcventry - uctxt->expected_base);  in tid_rb_invalidate()
943 			ev = uctxt->dd->events +  in tid_rb_invalidate()
944 				(uctxt_offset(uctxt) + fdata->subctxt);  in tid_rb_invalidate()
973 	u32 base = fdata->uctxt->expected_base;  in cacheless_tid_rb_remove()