15 			    struct hfi1_filedata *fd);
17 static int set_rcvarray_entry(struct hfi1_filedata *fd,
29 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
33 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);
34 static void __clear_tid_node(struct hfi1_filedata *fd,
36 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);
50 int hfi1_user_exp_rcv_init(struct hfi1_filedata *fd,  in hfi1_user_exp_rcv_init()  argument
55 	fd->entry_to_rb = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
58 	if (!fd->entry_to_rb)  in hfi1_user_exp_rcv_init()
62 		fd->invalid_tid_idx = 0;  in hfi1_user_exp_rcv_init()
63 		fd->invalid_tids = kcalloc(uctxt->expected_count,  in hfi1_user_exp_rcv_init()
64 					   sizeof(*fd->invalid_tids),  in hfi1_user_exp_rcv_init()
66 		if (!fd->invalid_tids) {  in hfi1_user_exp_rcv_init()
67 			kfree(fd->entry_to_rb);  in hfi1_user_exp_rcv_init()
68 			fd->entry_to_rb = NULL;  in hfi1_user_exp_rcv_init()
71 		fd->use_mn = true;  in hfi1_user_exp_rcv_init()
87 	spin_lock(&fd->tid_lock);  in hfi1_user_exp_rcv_init()
88 	if (uctxt->subctxt_cnt && fd->use_mn) {  in hfi1_user_exp_rcv_init()
91 		fd->tid_limit = uctxt->expected_count / uctxt->subctxt_cnt;  in hfi1_user_exp_rcv_init()
93 		if (remainder && fd->subctxt < remainder)  in hfi1_user_exp_rcv_init()
94 			fd->tid_limit++;  in hfi1_user_exp_rcv_init()
96 		fd->tid_limit = uctxt->expected_count;  in hfi1_user_exp_rcv_init()
98 	spin_unlock(&fd->tid_lock);  in hfi1_user_exp_rcv_init()
103 void hfi1_user_exp_rcv_free(struct hfi1_filedata *fd)  in hfi1_user_exp_rcv_free()  argument
105 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_free()
109 		unlock_exp_tids(uctxt, &uctxt->tid_full_list, fd);  in hfi1_user_exp_rcv_free()
111 		unlock_exp_tids(uctxt, &uctxt->tid_used_list, fd);  in hfi1_user_exp_rcv_free()
114 	kfree(fd->invalid_tids);  in hfi1_user_exp_rcv_free()
115 	fd->invalid_tids = NULL;  in hfi1_user_exp_rcv_free()
117 	kfree(fd->entry_to_rb);  in hfi1_user_exp_rcv_free()
118 	fd->entry_to_rb = NULL;  in hfi1_user_exp_rcv_free()
132 static void unpin_rcv_pages(struct hfi1_filedata *fd,  in unpin_rcv_pages()  argument
140 	struct hfi1_devdata *dd = fd->uctxt->dd;  in unpin_rcv_pages()
153 	fd->tid_n_pinned -= npages;  in unpin_rcv_pages()
159 static int pin_rcv_pages(struct hfi1_filedata *fd, struct tid_user_buf *tidbuf)  in pin_rcv_pages()  argument
165 	struct hfi1_devdata *dd = fd->uctxt->dd;  in pin_rcv_pages()
167 	if (npages > fd->uctxt->expected_count) {  in pin_rcv_pages()
182 	if (!hfi1_can_pin_pages(dd, current->mm, fd->tid_n_pinned, npages)) {  in pin_rcv_pages()
193 	fd->tid_n_pinned += pinned;  in pin_rcv_pages()
246 int hfi1_user_exp_rcv_setup(struct hfi1_filedata *fd,  in hfi1_user_exp_rcv_setup()  argument
250 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_setup()
278 	if (fd->use_mn) {  in hfi1_user_exp_rcv_setup()
288 	pinned = pin_rcv_pages(fd, tidbuf);  in hfi1_user_exp_rcv_setup()
298 	spin_lock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
299 	if (fd->tid_used + tidbuf->n_psets > fd->tid_limit)  in hfi1_user_exp_rcv_setup()
300 		pageset_count = fd->tid_limit - fd->tid_used;  in hfi1_user_exp_rcv_setup()
303 	fd->tid_used += pageset_count;  in hfi1_user_exp_rcv_setup()
304 	spin_unlock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
333 		ret = program_rcvarray(fd, tidbuf, grp,  in hfi1_user_exp_rcv_setup()
378 			ret = program_rcvarray(fd, tidbuf, grp,  in hfi1_user_exp_rcv_setup()
420 	spin_lock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
421 	fd->tid_used -= pageset_count - tididx;  in hfi1_user_exp_rcv_setup()
422 	spin_unlock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
425 	unpin_rcv_pages(fd, tidbuf, NULL, mapped_pages, pinned - mapped_pages,  in hfi1_user_exp_rcv_setup()
428 	if (fd->use_mn) {  in hfi1_user_exp_rcv_setup()
451 	if (fd->use_mn)  in hfi1_user_exp_rcv_setup()
462 	hfi1_user_exp_rcv_clear(fd, tinfo);  in hfi1_user_exp_rcv_setup()
467 	spin_lock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
468 	fd->tid_used -= pageset_count;  in hfi1_user_exp_rcv_setup()
469 	spin_unlock(&fd->tid_lock);  in hfi1_user_exp_rcv_setup()
471 	if (fd->use_mn)  in hfi1_user_exp_rcv_setup()
474 		unpin_rcv_pages(fd, tidbuf, NULL, 0, pinned, false);  in hfi1_user_exp_rcv_setup()
483 int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,  in hfi1_user_exp_rcv_clear()  argument
487 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_clear()
491 	if (unlikely(tinfo->tidcnt > fd->tid_used))  in hfi1_user_exp_rcv_clear()
501 		ret = unprogram_rcvarray(fd, tidinfo[tididx]);  in hfi1_user_exp_rcv_clear()
508 	spin_lock(&fd->tid_lock);  in hfi1_user_exp_rcv_clear()
509 	fd->tid_used -= tididx;  in hfi1_user_exp_rcv_clear()
510 	spin_unlock(&fd->tid_lock);  in hfi1_user_exp_rcv_clear()
518 int hfi1_user_exp_rcv_invalid(struct hfi1_filedata *fd,  in hfi1_user_exp_rcv_invalid()  argument
521 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in hfi1_user_exp_rcv_invalid()
523 		(uctxt_offset(uctxt) + fd->subctxt);  in hfi1_user_exp_rcv_invalid()
537 	spin_lock(&fd->invalid_lock);  in hfi1_user_exp_rcv_invalid()
538 	if (fd->invalid_tid_idx) {  in hfi1_user_exp_rcv_invalid()
539 		memcpy(array, fd->invalid_tids, sizeof(*array) *  in hfi1_user_exp_rcv_invalid()
540 		       fd->invalid_tid_idx);  in hfi1_user_exp_rcv_invalid()
541 		memset(fd->invalid_tids, 0, sizeof(*fd->invalid_tids) *  in hfi1_user_exp_rcv_invalid()
542 		       fd->invalid_tid_idx);  in hfi1_user_exp_rcv_invalid()
543 		tinfo->tidcnt = fd->invalid_tid_idx;  in hfi1_user_exp_rcv_invalid()
544 		fd->invalid_tid_idx = 0;  in hfi1_user_exp_rcv_invalid()
553 	spin_unlock(&fd->invalid_lock);  in hfi1_user_exp_rcv_invalid()
656 static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *tbuf,  in program_rcvarray()  argument
661 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in program_rcvarray()
702 		ret = set_rcvarray_entry(fd, tbuf,  in program_rcvarray()
723 static int set_rcvarray_entry(struct hfi1_filedata *fd,  in set_rcvarray_entry()  argument
729 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in set_rcvarray_entry()
752 	node->fdata = fd;  in set_rcvarray_entry()
762 	if (fd->use_mn) {  in set_rcvarray_entry()
770 	fd->entry_to_rb[node->rcventry - uctxt->expected_base] = node;  in set_rcvarray_entry()
773 	trace_hfi1_exp_tid_reg(uctxt->ctxt, fd->subctxt, rcventry, npages,  in set_rcvarray_entry()
788 static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo)  in unprogram_rcvarray()  argument
790 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in unprogram_rcvarray()
807 	node = fd->entry_to_rb[rcventry];  in unprogram_rcvarray()
811 	if (fd->use_mn)  in unprogram_rcvarray()
813 	cacheless_tid_rb_remove(fd, node);  in unprogram_rcvarray()
818 static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)  in __clear_tid_node()  argument
820 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in __clear_tid_node()
828 	trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,  in __clear_tid_node()
836 	unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);  in __clear_tid_node()
841 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)  in clear_tid_node()  argument
843 	struct hfi1_ctxtdata *uctxt = fd->uctxt;  in clear_tid_node()
845 	__clear_tid_node(fd, node);  in clear_tid_node()
865 			    struct hfi1_filedata *fd)  in unlock_exp_tids()  argument
878 				node = fd->entry_to_rb[rcventry -  in unlock_exp_tids()
883 				if (fd->use_mn)  in unlock_exp_tids()
886 				cacheless_tid_rb_remove(fd, node);  in unlock_exp_tids()